haylan/git-contribution-graph
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

feat(store): fetch contributions incrementally and persist history in SQLite

Browse Source 
Bound each provider's startFetch()/resolveFetch() to an optional
since/until window, wire a SQLite-backed ContributionStore into
ContributionAggregator (fetch only the trailing window past the last
stored date, merge into the store, prune by CONTRIBUTIONS_RETENTION_DAYS),
and add a graph:contributions:refetch command to force a full or ranged
re-fetch in <=365-day chunks. This is the root fix for the full
365-day-refetch timeout that used to hit on every cache miss.
This commit is contained in:
haylan
2026-07-12 23:42:41 +02:00
parent 41e88144a4
commit 4ff45b7c46
16 changed files with 642 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Command/RefetchContributionsCommand.php
+157 -20
View File
@@ -4,51 +4,188 @@ declare(strict_types=1);
namespace GitContributionGraph\Command;
use GitContributionGraph\Service\ContributionStore;
use GitContributionGraph\Service\Provider\ProviderInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Attribute\Option;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use Symfony\Component\DependencyInjection\Attribute\AutowireIterator;
#[AsCommand(name: 'graph:contributions:refetch')]
class RefetchContributionsCommand extends Command
#[AsCommand(
name: 'graph:contributions:refetch',
description: 'Force a full or ranged re-fetch of contribution history into the store.',
)]
final class RefetchContributionsCommand extends Command
{
private const MAX_CHUNK_DAYS = 365;
private const ALL_SINCE = '2005-01-01';
private SymfonyStyle $io;
/** @param iterable<ProviderInterface> $providers */
public function __construct(
#[AutowireIterator('app.provider')]
private readonly iterable $providers,
private readonly ContributionStore $store,
) {
parent::__construct();
}
public function initialize(InputInterface $input, OutputInterface $output)
/** Symfony console lifecycle hook: sets up the styled I/O helper used throughout the command. */
public function initialize(InputInterface $input, OutputInterface $output): void
{
$this->io = new SymfonyStyle($input, $output);
}
/**
* Bypasses the aggregator's incremental trailing-window fetch to re-pull a full or explicit
* date range for one, several, or all configured providers, chunked into ≤365-day windows.
*
* @param string $provider comma-separated provider names to refetch; empty = all configured
* @param string $from start date (YYYY-MM-DD); empty defaults to 365 days ago
* @param string $to end date (YYYY-MM-DD); empty defaults to today
* @param bool $all shorthand for --from=2005-01-01
* @return int Command::SUCCESS if at least one provider refetched cleanly, else Command::FAILURE
*/
public function __invoke(
//TODO: check if its possible to add the preview with real strings from the providers.
#[Option("(repeatable/comma-split, restricts to named provider(s), default = all configured", "provider", "p", "")]
string $providers = ""
) {
$requested = $providers === '' ? null : explode(',', $providers);
$byName = [];
foreach ($this->providers as $provider) {
$byName[$provider->getName()] = $provider;
#[Option('Comma-separated provider names to refetch (default: all configured)', 'provider', 'p')]
string $provider = '',
#[Option('Start date (YYYY-MM-DD), default: 365 days ago', 'from')]
string $from = '',
#[Option('End date (YYYY-MM-DD), default: today', 'to')]
string $to = '',
#[Option('Refetch full history (from ' . self::ALL_SINCE . ')', 'all')]
bool $all = false,
): int {
$byName = $this->resolveProviders($provider);
if ($byName === null) {
return Command::FAILURE;
}
if ($requested !== null) {
foreach ($requested as $name) {
if (!isset($byName[$name])) {
$this->io->error("Unknown provider: {$name}");
return Command::FAILURE;
}
$fromSpec = match (true) {
$all => self::ALL_SINCE,
$from !== '' => $from,
default => '-365 days',
};
$fromDate = new \DateTimeImmutable($fromSpec);
$toDate = new \DateTimeImmutable($to !== '' ? $to : 'today');
$chunks = $this->chunk($fromDate, $toDate);
$summary = [];
$succeeded = 0;
foreach ($byName as $name => $providerInstance) {
[$daysWritten, $errors] = $this->refetchProvider($name, $providerInstance, $chunks);
if ($errors === 0) {
$succeeded++;
}
$byName = array_intersect_key($byName, array_flip($requested));
$summary[] = [$name, $daysWritten, count($chunks), $errors];
}
$this->io->table(['Provider', 'Days written', 'Chunks fetched', 'Errors'], $summary);
if ($succeeded === 0) {
$this->io->error('No provider completed successfully.');
return Command::FAILURE;
}
$this->io->success(sprintf('%d of %d provider(s) refetched successfully.', $succeeded, count($byName)));
return Command::SUCCESS;
}
/**
* @param list<array{0: \DateTimeImmutable, 1: \DateTimeImmutable}> $chunks
*
* @return array{0: int, 1: int} days written, errors
*/
private function refetchProvider(string $name, ProviderInterface $provider, array $chunks): array
{
$this->io->section("Refetching {$name}");
$progress = new ProgressBar($this->io, count($chunks));
$progress->start();
$daysWritten = 0;
$errors = 0;
foreach ($chunks as [$chunkFrom, $chunkTo]) {
try {
$fresh = $provider->resolveFetch($provider->startFetch($chunkFrom, $chunkTo));
$dateCounts = [];
foreach ($fresh as $date => $count) {
$dateCounts[(new \DateTimeImmutable($date))->getTimestamp()] = $count;
}
$this->store->merge($name, $dateCounts);
$daysWritten += count($dateCounts);
} catch (\Throwable $e) {
$errors++;
$this->io->warning("{$name}: {$e->getMessage()}");
}
$progress->advance();
}
$progress->finish();
$this->io->newLine(2);
return [$daysWritten, $errors];
}
/**
* @return array<string, ProviderInterface>|null null when an unknown provider name was requested
*/
private function resolveProviders(string $requested): ?array
{
$byName = [];
foreach ($this->providers as $providerInstance) {
$byName[$providerInstance->getName()] = $providerInstance;
}
if ($requested === '') {
return array_filter($byName, static fn (ProviderInterface $p): bool => $p->isConfigured());
}
$names = explode(',', $requested);
foreach ($names as $name) {
if (!isset($byName[$name])) {
$this->io->error("Unknown provider: {$name}");
return null;
}
}
return array_intersect_key($byName, array_flip($names));
}
/**
* @return list<array{0: \DateTimeImmutable, 1: \DateTimeImmutable}>
*/
private function chunk(\DateTimeImmutable $from, \DateTimeImmutable $to): array
{
$chunks = [];
$cursor = $from;
while ($cursor < $to) {
$chunkEnd = min($cursor->modify('+' . (self::MAX_CHUNK_DAYS - 1) . ' days'), $to);
$chunks[] = [$cursor, $chunkEnd];
$cursor = $chunkEnd->modify('+1 day');
}
if ($chunks === []) {
$chunks[] = [$from, $to];
}
return $chunks;
}
}
src/Service/ContributionAggregator.php
+50 -12
View File
@@ -10,25 +10,64 @@ use Symfony\Component\DependencyInjection\Attribute\AutowireIterator;
final class ContributionAggregator
{
private const RENDER_WINDOW_DAYS = 371;
private const OVERLAP_DAYS = 3;
/** @param iterable<ProviderInterface> $providers */
public function __construct(
#[AutowireIterator('app.provider')]
private readonly iterable $providers,
private readonly ContributionStore $store,
private readonly LoggerInterface $logger,
) {}
/** @return array<string, int> */
/**
* Incrementally fetches fresh contributions from each configured provider, merges them into
* the store, and returns the rolling render window summed across providers.
*
* For each provider, only data since its last stored date (minus a trailing overlap) is
* fetched; a provider with no stored history yet fetches its full default range. Fetches are
* split into start/resolve phases so all providers' requests are in flight concurrently.
* Failures are logged and the affected provider is skipped, not fatal to the others.
*
* @return array<string, int> date (Y-m-d) => contribution count, summed across all providers
*/
public function aggregate(): array
{
$pending = [];
$configured = [];
/** @var ProviderInterface $provider */
foreach ($this->providers as $provider) {
if (!$provider->isConfigured()) {
continue;
if ($provider->isConfigured()) {
$configured[] = $provider;
}
}
$pending = [];
foreach ($configured as $provider) {
$latest = $this->store->latestDate($provider->getName());
$since = $latest !== null
? (new \DateTimeImmutable('@' . $latest))->modify('-' . self::OVERLAP_DAYS . ' days')
: null;
try {
$pending[] = [$provider, $provider->startFetch()];
$pending[] = [$provider, $provider->startFetch($since)];
} catch (\Throwable $e) {
$this->logger->warning(sprintf('%s fetch failed: %s', $provider::class, $e->getMessage()), ['exception' => $e]);
}
}
foreach ($pending as [$provider, $handle]) {
try {
$fresh = $provider->resolveFetch($handle);
$dateCounts = [];
foreach ($fresh as $date => $count) {
$dateCounts[(new \DateTimeImmutable($date))->getTimestamp()] = $count;
}
$this->store->merge($provider->getName(), $dateCounts);
} catch (\Throwable $e) {
$this->logger->warning(sprintf('%s fetch failed: %s', $provider::class, $e->getMessage()), ['exception' => $e]);
}
@@ -36,16 +75,15 @@ final class ContributionAggregator
$contributions = [];
foreach ($pending as [$provider, $handle]) {
try {
foreach ($provider->resolveFetch($handle) as $date => $count) {
$contributions[$date] = ($contributions[$date] ?? 0) + $count;
}
} catch (\Throwable $e) {
$this->logger->warning(sprintf('%s fetch failed: %s', $provider::class, $e->getMessage()), ['exception' => $e]);
foreach ($configured as $provider) {
foreach ($this->store->all($provider->getName(), sinceDays: self::RENDER_WINDOW_DAYS) as $contribution) {
$date = (new \DateTimeImmutable('@' . $contribution->date))->format('Y-m-d');
$contributions[$date] = ($contributions[$date] ?? 0) + $contribution->count;
}
}
$this->store->prune();
return $contributions;
}
}
src/Service/Provider/GitHubProvider.php
+8 -4
View File
@@ -44,12 +44,13 @@ final class GitHubProvider implements ProviderInterface
])->getContent();
}
public function startFetch(): ResponseInterface
/** Fires the GraphQL contributionsCollection query bounded by $since/$until. */
public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): ResponseInterface
{
$this->logger->debug('GitHubProvider: fetching contributions', ['user' => $this->username]);
$from = (new \DateTimeImmutable('-365 days'))->format('Y-m-d\T00:00:00\Z');
$to = (new \DateTimeImmutable())->format('Y-m-d\T23:59:59\Z');
$from = ($since ?? new \DateTimeImmutable('-365 days'))->format('Y-m-d\T00:00:00\Z');
$to = ($until ?? new \DateTimeImmutable())->format('Y-m-d\T23:59:59\Z');
$query = sprintf(
'query { user(login: %s) { contributionsCollection(from: %s, to: %s) { contributionCalendar { weeks { contributionDays { date contributionCount } } } } } }',
@@ -71,7 +72,10 @@ final class GitHubProvider implements ProviderInterface
}
/**
* @return array<string, int> date (Y-m-d) => contribution count
* Parses the GraphQL response into a per-day count map, skipping zero-count days.
*
* @param ResponseInterface $handle the response returned by startFetch()
* @return array<string, int> date (Y-m-d) => contribution count
*/
public function resolveFetch(mixed $handle): array
{
src/Service/Provider/GitLabProvider.php
+25 -17
View File
@@ -50,16 +50,18 @@ final class GitLabProvider implements ProviderInterface
}
/**
* @return array{baseUrl: string, userId: int, after: string, page: int, response: ResponseInterface}
* Resolves the username to a numeric user ID, then fires the first page of the events request.
*
* @return array{baseUrl: string, userId: int, after: string, before: ?string, page: int, response: ResponseInterface}
*
* ponytail: the user-id lookup and event pagination stay sequential within
* this one provider (each page depends on the previous). Parallelism here
* only spans across providers; revisit only if GitLab pagination itself
* becomes the bottleneck.
*/
public function startFetch(): array
public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): array
{
$this->logger->debug('GitLabProvider: fetching contributions', ['user' => $this->username, 'url' => $this->baseUrl]);
$userResponse = $this->client->request('GET', "$this->baseUrl/api/v4/users", [
@@ -72,26 +74,31 @@ final class GitLabProvider implements ProviderInterface
throw new NotFoundHttpException("GitLab: user '{$this->username}' not found on $this->baseUrl");
}
$userId = $users[0]['id'];
$after = (new \DateTimeImmutable('-365 days'))->format('Y-m-d');
$after = ($since ?? new \DateTimeImmutable('-365 days'))->format('Y-m-d');
$before = $until?->format('Y-m-d');
$query = ['after' => $after, 'per_page' => 100, 'page' => 1];
if ($before !== null) {
$query['before'] = $before;
}
$response = $this->client->request('GET', "$this->baseUrl/api/v4/users/$userId/events", [
'headers' => ['PRIVATE-TOKEN' => $this->token],
'query' => [
'after' => $after,
'per_page' => 100,
'page' => 1,
],
'query' => $query,
]);
return ['baseUrl' => $this->baseUrl, 'userId' => $userId, 'after' => $after, 'page' => 1, 'response' => $response];
return ['baseUrl' => $this->baseUrl, 'userId' => $userId, 'after' => $after, 'before' => $before, 'page' => 1, 'response' => $response];
}
/**
* @return array<string, int> date (Y-m-d) => event count
* Follows pagination (100 events/page) until a short page is returned, counting events per day.
*
* @param array{baseUrl: string, userId: int, after: string, before: ?string, page: int, response: ResponseInterface} $handle
* @return array<string, int> date (Y-m-d) => event count
*/
public function resolveFetch(mixed $handle): array
{
['baseUrl' => $baseUrl, 'userId' => $userId, 'after' => $after, 'page' => $page, 'response' => $response] = $handle;
['baseUrl' => $baseUrl, 'userId' => $userId, 'after' => $after, 'before' => $before, 'page' => $page, 'response' => $response] = $handle;
$result = [];
@@ -106,13 +113,14 @@ final class GitLabProvider implements ProviderInterface
$page++;
if (count($events) === 100) {
$query = ['after' => $after, 'per_page' => 100, 'page' => $page];
if ($before !== null) {
$query['before'] = $before;
}
$response = $this->client->request('GET', "$baseUrl/api/v4/users/$userId/events", [
'headers' => ['PRIVATE-TOKEN' => $this->token],
'query' => [
'after' => $after,
'per_page' => 100,
'page' => $page,
],
'query' => $query,
]);
}
} while (count($events) === 100);
src/Service/Provider/GiteaProvider.php
+23 -9
View File
@@ -51,31 +51,45 @@ final class GiteaProvider implements ProviderInterface
])->getContent();
}
public function startFetch(): ResponseInterface
/**
* Fires a single heatmap request; $since/$until are carried through unused for client-side filtering in resolveFetch().
*
* @return array{response: ResponseInterface, since: ?\DateTimeImmutable, until: ?\DateTimeImmutable}
*/
public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): array
{
$this->logger->debug('GiteaProvider: fetching contributions', ['user' => $this->username, 'url' => $this->baseUrl]);
return $this->client->request('GET', "$this->baseUrl/api/v1/users/{$this->username}/heatmap", [
$response = $this->client->request('GET', "$this->baseUrl/api/v1/users/{$this->username}/heatmap", [
'headers' => ['Authorization' => "token {$this->token}"],
]);
return ['response' => $response, 'since' => $since, 'until' => $until];
}
/**
* @return array<string, int> date (Y-m-d) => contribution count
* Filters the heatmap entries to the [$since, $until] window and sums contributions per day.
*
* @param array{response: ResponseInterface, since: ?\DateTimeImmutable, until: ?\DateTimeImmutable} $handle
* @return array<string, int> date (Y-m-d) => contribution count
*/
public function resolveFetch(mixed $handle): array
{
/** @var ResponseInterface $handle */
$data = $handle->toArray();
$cutoff = (new \DateTimeImmutable('-365 days'))->getTimestamp();
$result = [];
['response' => $response, 'since' => $since, 'until' => $until] = $handle;
/** @var ResponseInterface $response */
$data = $response->toArray();
$cutoff = ($since ?? new \DateTimeImmutable('-365 days'))->getTimestamp();
$ceiling = $until?->getTimestamp();
$result = [];
foreach ($data as $entry) {
if ($entry['timestamp'] < $cutoff) {
continue;
}
if ($ceiling !== null && $entry['timestamp'] > $ceiling) {
continue;
}
$date = date('Y-m-d', $entry['timestamp']);
$result[$date] = ($result[$date] ?? 0) + (int) $entry['contributions'];
}
src/Service/Provider/ProviderInterface.php
+19 -3
View File
@@ -6,17 +6,33 @@ namespace GitContributionGraph\Service\Provider;
interface ProviderInterface
{
/** Fire the HTTP request(s) without blocking; returns an opaque handle for resolveFetch(). */
public function startFetch(): mixed;
/**
* Fire the HTTP request(s) without blocking; the result is read later by
* resolveFetch() so multiple providers' requests can be in flight at once.
*
* @param ?\DateTimeImmutable $since lower bound (inclusive); defaults to 365 days ago
* @param ?\DateTimeImmutable $until upper bound (inclusive); defaults to now
* @return mixed opaque handle to pass into resolveFetch()
*/
public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): mixed;
/** @return array<string, int> date (Y-m-d) => contribution count */
/**
* Block on the handle from startFetch() and parse it into a per-day count map.
*
* @param mixed $handle the value returned by startFetch()
* @return array<string, int> date (Y-m-d) => contribution count
*/
public function resolveFetch(mixed $handle): array;
/** Whether the required credentials/URL for this provider are all set. */
public function isConfigured(): bool;
/** Short lowercase identifier for this provider, e.g. "github". */
public function getName(): string;
/** Check reachability/credentials without fetching contribution data; used by the /health endpoint. */
public function probe(): ProviderStatus;
/** Make a lightweight authenticated request to verify the provider is reachable; throws on failure. */
public function ping(): void;
}