From 4ff45b7c46c5da824fe45f9ae66ab4fb78d86926 Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sun, 12 Jul 2026 23:42:41 +0200 Subject: [PATCH] feat(store): fetch contributions incrementally and persist history in SQLite Bound each provider's startFetch()/resolveFetch() to an optional since/until window, wire a SQLite-backed ContributionStore into ContributionAggregator (fetch only the trailing window past the last stored date, merge into the store, prune by CONTRIBUTIONS_RETENTION_DAYS), and add a graph:contributions:refetch command to force a full or ranged re-fetch in <=365-day chunks. This is the root fix for the full 365-day-refetch timeout that used to hit on every cache miss. --- .env | 4 + Dockerfile | 2 +- config/services.yaml | 1 + docker-compose.yml | 3 + src/Command/RefetchContributionsCommand.php | 177 ++++++++++++++++-- src/Service/ContributionAggregator.php | 62 ++++-- src/Service/Provider/GitHubProvider.php | 12 +- src/Service/Provider/GitLabProvider.php | 42 +++-- src/Service/Provider/GiteaProvider.php | 32 +++- src/Service/Provider/ProviderInterface.php | 22 ++- .../RefetchContributionsCommandTest.php | 149 +++++++++++++++ .../Service/ContributionAggregatorTest.php | 141 +++++++++++--- .../Service/Provider/GitHubProviderTest.php | 24 +++ .../Service/Provider/GitLabProviderTest.php | 24 +++ .../Service/Provider/GiteaProviderTest.php | 41 ++++ .../Unit/Service/Provider/ProbeTraitTest.php | 2 +- 16 files changed, 642 insertions(+), 96 deletions(-) create mode 100644 tests/Unit/Command/RefetchContributionsCommandTest.php diff --git a/.env b/.env index 008ef58..57fdf5e 100644 --- a/.env +++ b/.env @@ -19,3 +19,7 @@ GITLAB_URL= GITEA_USER= GITEA_TOKEN= GITEA_URL= + +# Number of days of contribution history to keep in var/data/contributions.db. +# 0 (or unset) keeps history forever. +CONTRIBUTIONS_RETENTION_DAYS=0 diff --git a/Dockerfile b/Dockerfile index bc313f6..6abde98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ COPY --link docker/frankenphp/Caddyfile /etc/caddy/Caddyfile COPY --link docker/php/conf.d/20-app.prod.ini $PHP_INI_DIR/app.conf.d/ RUN chmod +x bin/console && \ - mkdir -p var/cache/prod/pools var/log /config/caddy /data/caddy && \ + mkdir -p var/cache/prod/pools var/log var/data /config/caddy /data/caddy && \ chown -R app:app /app /config /data USER app diff --git a/config/services.yaml b/config/services.yaml index fccfd34..4f76ae9 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -9,6 +9,7 @@ parameters: env(GITEA_USER): "" env(GITEA_TOKEN): "" env(GITEA_URL): "" + env(CONTRIBUTIONS_RETENTION_DAYS): "0" services: _defaults: diff --git a/docker-compose.yml b/docker-compose.yml index 81d382c..98a613a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,9 +20,11 @@ services: GITEA_USER: "${GITEA_USER:-}" GITEA_TOKEN: "${GITEA_TOKEN:-}" GITEA_URL: "${GITEA_URL:-}" + CONTRIBUTIONS_RETENTION_DAYS: "${CONTRIBUTIONS_RETENTION_DAYS:-0}" volumes: - cache:/app/var/cache/prod/pools - logs:/app/var/log + - data:/app/var/data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/health"] interval: 30s @@ -33,3 +35,4 @@ services: volumes: cache: logs: + data: diff --git a/src/Command/RefetchContributionsCommand.php b/src/Command/RefetchContributionsCommand.php index 8724a6e..8935ba2 100644 --- a/src/Command/RefetchContributionsCommand.php +++ b/src/Command/RefetchContributionsCommand.php @@ -4,51 +4,188 @@ declare(strict_types=1); namespace GitContributionGraph\Command; +use GitContributionGraph\Service\ContributionStore; +use GitContributionGraph\Service\Provider\ProviderInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Attribute\Option; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; use Symfony\Component\DependencyInjection\Attribute\AutowireIterator; -#[AsCommand(name: 'graph:contributions:refetch')] -class RefetchContributionsCommand extends Command +#[AsCommand( + name: 'graph:contributions:refetch', + description: 'Force a full or ranged re-fetch of contribution history into the store.', +)] +final class RefetchContributionsCommand extends Command { + private const MAX_CHUNK_DAYS = 365; + private const ALL_SINCE = '2005-01-01'; + private SymfonyStyle $io; + + /** @param iterable $providers */ public function __construct( #[AutowireIterator('app.provider')] private readonly iterable $providers, + private readonly ContributionStore $store, ) { parent::__construct(); } - - public function initialize(InputInterface $input, OutputInterface $output) + /** Symfony console lifecycle hook: sets up the styled I/O helper used throughout the command. */ + public function initialize(InputInterface $input, OutputInterface $output): void { $this->io = new SymfonyStyle($input, $output); } + /** + * Bypasses the aggregator's incremental trailing-window fetch to re-pull a full or explicit + * date range for one, several, or all configured providers, chunked into ≤365-day windows. + * + * @param string $provider comma-separated provider names to refetch; empty = all configured + * @param string $from start date (YYYY-MM-DD); empty defaults to 365 days ago + * @param string $to end date (YYYY-MM-DD); empty defaults to today + * @param bool $all shorthand for --from=2005-01-01 + * @return int Command::SUCCESS if at least one provider refetched cleanly, else Command::FAILURE + */ public function __invoke( - //TODO: check if its possible to add the preview with real strings from the providers. - #[Option("(repeatable/comma-split, restricts to named provider(s), default = all configured", "provider", "p", "")] - string $providers = "" - ) { - $requested = $providers === '' ? null : explode(',', $providers); - $byName = []; - - foreach ($this->providers as $provider) { - $byName[$provider->getName()] = $provider; + #[Option('Comma-separated provider names to refetch (default: all configured)', 'provider', 'p')] + string $provider = '', + #[Option('Start date (YYYY-MM-DD), default: 365 days ago', 'from')] + string $from = '', + #[Option('End date (YYYY-MM-DD), default: today', 'to')] + string $to = '', + #[Option('Refetch full history (from ' . self::ALL_SINCE . ')', 'all')] + bool $all = false, + ): int { + $byName = $this->resolveProviders($provider); + if ($byName === null) { + return Command::FAILURE; } - if ($requested !== null) { - foreach ($requested as $name) { - if (!isset($byName[$name])) { - $this->io->error("Unknown provider: {$name}"); - return Command::FAILURE; - } + $fromSpec = match (true) { + $all => self::ALL_SINCE, + $from !== '' => $from, + default => '-365 days', + }; + + $fromDate = new \DateTimeImmutable($fromSpec); + $toDate = new \DateTimeImmutable($to !== '' ? $to : 'today'); + $chunks = $this->chunk($fromDate, $toDate); + + $summary = []; + $succeeded = 0; + + foreach ($byName as $name => $providerInstance) { + [$daysWritten, $errors] = $this->refetchProvider($name, $providerInstance, $chunks); + + if ($errors === 0) { + $succeeded++; } - $byName = array_intersect_key($byName, array_flip($requested)); + + $summary[] = [$name, $daysWritten, count($chunks), $errors]; } + + $this->io->table(['Provider', 'Days written', 'Chunks fetched', 'Errors'], $summary); + + if ($succeeded === 0) { + $this->io->error('No provider completed successfully.'); + + return Command::FAILURE; + } + + $this->io->success(sprintf('%d of %d provider(s) refetched successfully.', $succeeded, count($byName))); + + return Command::SUCCESS; + } + + /** + * @param list $chunks + * + * @return array{0: int, 1: int} days written, errors + */ + private function refetchProvider(string $name, ProviderInterface $provider, array $chunks): array + { + $this->io->section("Refetching {$name}"); + + $progress = new ProgressBar($this->io, count($chunks)); + $progress->start(); + + $daysWritten = 0; + $errors = 0; + + foreach ($chunks as [$chunkFrom, $chunkTo]) { + try { + $fresh = $provider->resolveFetch($provider->startFetch($chunkFrom, $chunkTo)); + + $dateCounts = []; + foreach ($fresh as $date => $count) { + $dateCounts[(new \DateTimeImmutable($date))->getTimestamp()] = $count; + } + + $this->store->merge($name, $dateCounts); + $daysWritten += count($dateCounts); + } catch (\Throwable $e) { + $errors++; + $this->io->warning("{$name}: {$e->getMessage()}"); + } + + $progress->advance(); + } + + $progress->finish(); + $this->io->newLine(2); + + return [$daysWritten, $errors]; + } + + /** + * @return array|null null when an unknown provider name was requested + */ + private function resolveProviders(string $requested): ?array + { + $byName = []; + foreach ($this->providers as $providerInstance) { + $byName[$providerInstance->getName()] = $providerInstance; + } + + if ($requested === '') { + return array_filter($byName, static fn (ProviderInterface $p): bool => $p->isConfigured()); + } + + $names = explode(',', $requested); + foreach ($names as $name) { + if (!isset($byName[$name])) { + $this->io->error("Unknown provider: {$name}"); + + return null; + } + } + + return array_intersect_key($byName, array_flip($names)); + } + + /** + * @return list + */ + private function chunk(\DateTimeImmutable $from, \DateTimeImmutable $to): array + { + $chunks = []; + $cursor = $from; + + while ($cursor < $to) { + $chunkEnd = min($cursor->modify('+' . (self::MAX_CHUNK_DAYS - 1) . ' days'), $to); + $chunks[] = [$cursor, $chunkEnd]; + $cursor = $chunkEnd->modify('+1 day'); + } + + if ($chunks === []) { + $chunks[] = [$from, $to]; + } + + return $chunks; } } diff --git a/src/Service/ContributionAggregator.php b/src/Service/ContributionAggregator.php index 4bdd2f7..7aed7ce 100644 --- a/src/Service/ContributionAggregator.php +++ b/src/Service/ContributionAggregator.php @@ -10,25 +10,64 @@ use Symfony\Component\DependencyInjection\Attribute\AutowireIterator; final class ContributionAggregator { + private const RENDER_WINDOW_DAYS = 371; + private const OVERLAP_DAYS = 3; + + /** @param iterable $providers */ public function __construct( #[AutowireIterator('app.provider')] private readonly iterable $providers, + private readonly ContributionStore $store, private readonly LoggerInterface $logger, ) {} - /** @return array */ + /** + * Incrementally fetches fresh contributions from each configured provider, merges them into + * the store, and returns the rolling render window summed across providers. + * + * For each provider, only data since its last stored date (minus a trailing overlap) is + * fetched; a provider with no stored history yet fetches its full default range. Fetches are + * split into start/resolve phases so all providers' requests are in flight concurrently. + * Failures are logged and the affected provider is skipped, not fatal to the others. + * + * @return array date (Y-m-d) => contribution count, summed across all providers + */ public function aggregate(): array { - $pending = []; + $configured = []; /** @var ProviderInterface $provider */ foreach ($this->providers as $provider) { - if (!$provider->isConfigured()) { - continue; + if ($provider->isConfigured()) { + $configured[] = $provider; } + } + + $pending = []; + + foreach ($configured as $provider) { + $latest = $this->store->latestDate($provider->getName()); + $since = $latest !== null + ? (new \DateTimeImmutable('@' . $latest))->modify('-' . self::OVERLAP_DAYS . ' days') + : null; try { - $pending[] = [$provider, $provider->startFetch()]; + $pending[] = [$provider, $provider->startFetch($since)]; + } catch (\Throwable $e) { + $this->logger->warning(sprintf('%s fetch failed: %s', $provider::class, $e->getMessage()), ['exception' => $e]); + } + } + + foreach ($pending as [$provider, $handle]) { + try { + $fresh = $provider->resolveFetch($handle); + + $dateCounts = []; + foreach ($fresh as $date => $count) { + $dateCounts[(new \DateTimeImmutable($date))->getTimestamp()] = $count; + } + + $this->store->merge($provider->getName(), $dateCounts); } catch (\Throwable $e) { $this->logger->warning(sprintf('%s fetch failed: %s', $provider::class, $e->getMessage()), ['exception' => $e]); } @@ -36,16 +75,15 @@ final class ContributionAggregator $contributions = []; - foreach ($pending as [$provider, $handle]) { - try { - foreach ($provider->resolveFetch($handle) as $date => $count) { - $contributions[$date] = ($contributions[$date] ?? 0) + $count; - } - } catch (\Throwable $e) { - $this->logger->warning(sprintf('%s fetch failed: %s', $provider::class, $e->getMessage()), ['exception' => $e]); + foreach ($configured as $provider) { + foreach ($this->store->all($provider->getName(), sinceDays: self::RENDER_WINDOW_DAYS) as $contribution) { + $date = (new \DateTimeImmutable('@' . $contribution->date))->format('Y-m-d'); + $contributions[$date] = ($contributions[$date] ?? 0) + $contribution->count; } } + $this->store->prune(); + return $contributions; } } diff --git a/src/Service/Provider/GitHubProvider.php b/src/Service/Provider/GitHubProvider.php index 99f69fb..2a06968 100644 --- a/src/Service/Provider/GitHubProvider.php +++ b/src/Service/Provider/GitHubProvider.php @@ -44,12 +44,13 @@ final class GitHubProvider implements ProviderInterface ])->getContent(); } - public function startFetch(): ResponseInterface + /** Fires the GraphQL contributionsCollection query bounded by $since/$until. */ + public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): ResponseInterface { $this->logger->debug('GitHubProvider: fetching contributions', ['user' => $this->username]); - $from = (new \DateTimeImmutable('-365 days'))->format('Y-m-d\T00:00:00\Z'); - $to = (new \DateTimeImmutable())->format('Y-m-d\T23:59:59\Z'); + $from = ($since ?? new \DateTimeImmutable('-365 days'))->format('Y-m-d\T00:00:00\Z'); + $to = ($until ?? new \DateTimeImmutable())->format('Y-m-d\T23:59:59\Z'); $query = sprintf( 'query { user(login: %s) { contributionsCollection(from: %s, to: %s) { contributionCalendar { weeks { contributionDays { date contributionCount } } } } } }', @@ -71,7 +72,10 @@ final class GitHubProvider implements ProviderInterface } /** - * @return array date (Y-m-d) => contribution count + * Parses the GraphQL response into a per-day count map, skipping zero-count days. + * + * @param ResponseInterface $handle the response returned by startFetch() + * @return array date (Y-m-d) => contribution count */ public function resolveFetch(mixed $handle): array { diff --git a/src/Service/Provider/GitLabProvider.php b/src/Service/Provider/GitLabProvider.php index 3d8d07c..90ee566 100644 --- a/src/Service/Provider/GitLabProvider.php +++ b/src/Service/Provider/GitLabProvider.php @@ -50,16 +50,18 @@ final class GitLabProvider implements ProviderInterface } /** - * @return array{baseUrl: string, userId: int, after: string, page: int, response: ResponseInterface} + * Resolves the username to a numeric user ID, then fires the first page of the events request. + * + * @return array{baseUrl: string, userId: int, after: string, before: ?string, page: int, response: ResponseInterface} * * ponytail: the user-id lookup and event pagination stay sequential within * this one provider (each page depends on the previous). Parallelism here * only spans across providers; revisit only if GitLab pagination itself * becomes the bottleneck. */ - public function startFetch(): array + public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): array { - + $this->logger->debug('GitLabProvider: fetching contributions', ['user' => $this->username, 'url' => $this->baseUrl]); $userResponse = $this->client->request('GET', "$this->baseUrl/api/v4/users", [ @@ -72,26 +74,31 @@ final class GitLabProvider implements ProviderInterface throw new NotFoundHttpException("GitLab: user '{$this->username}' not found on $this->baseUrl"); } $userId = $users[0]['id']; - $after = (new \DateTimeImmutable('-365 days'))->format('Y-m-d'); + $after = ($since ?? new \DateTimeImmutable('-365 days'))->format('Y-m-d'); + $before = $until?->format('Y-m-d'); + + $query = ['after' => $after, 'per_page' => 100, 'page' => 1]; + if ($before !== null) { + $query['before'] = $before; + } $response = $this->client->request('GET', "$this->baseUrl/api/v4/users/$userId/events", [ 'headers' => ['PRIVATE-TOKEN' => $this->token], - 'query' => [ - 'after' => $after, - 'per_page' => 100, - 'page' => 1, - ], + 'query' => $query, ]); - return ['baseUrl' => $this->baseUrl, 'userId' => $userId, 'after' => $after, 'page' => 1, 'response' => $response]; + return ['baseUrl' => $this->baseUrl, 'userId' => $userId, 'after' => $after, 'before' => $before, 'page' => 1, 'response' => $response]; } /** - * @return array date (Y-m-d) => event count + * Follows pagination (100 events/page) until a short page is returned, counting events per day. + * + * @param array{baseUrl: string, userId: int, after: string, before: ?string, page: int, response: ResponseInterface} $handle + * @return array date (Y-m-d) => event count */ public function resolveFetch(mixed $handle): array { - ['baseUrl' => $baseUrl, 'userId' => $userId, 'after' => $after, 'page' => $page, 'response' => $response] = $handle; + ['baseUrl' => $baseUrl, 'userId' => $userId, 'after' => $after, 'before' => $before, 'page' => $page, 'response' => $response] = $handle; $result = []; @@ -106,13 +113,14 @@ final class GitLabProvider implements ProviderInterface $page++; if (count($events) === 100) { + $query = ['after' => $after, 'per_page' => 100, 'page' => $page]; + if ($before !== null) { + $query['before'] = $before; + } + $response = $this->client->request('GET', "$baseUrl/api/v4/users/$userId/events", [ 'headers' => ['PRIVATE-TOKEN' => $this->token], - 'query' => [ - 'after' => $after, - 'per_page' => 100, - 'page' => $page, - ], + 'query' => $query, ]); } } while (count($events) === 100); diff --git a/src/Service/Provider/GiteaProvider.php b/src/Service/Provider/GiteaProvider.php index 3863203..3e29d2a 100644 --- a/src/Service/Provider/GiteaProvider.php +++ b/src/Service/Provider/GiteaProvider.php @@ -51,31 +51,45 @@ final class GiteaProvider implements ProviderInterface ])->getContent(); } - public function startFetch(): ResponseInterface + /** + * Fires a single heatmap request; $since/$until are carried through unused for client-side filtering in resolveFetch(). + * + * @return array{response: ResponseInterface, since: ?\DateTimeImmutable, until: ?\DateTimeImmutable} + */ + public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): array { - - $this->logger->debug('GiteaProvider: fetching contributions', ['user' => $this->username, 'url' => $this->baseUrl]); - return $this->client->request('GET', "$this->baseUrl/api/v1/users/{$this->username}/heatmap", [ + $response = $this->client->request('GET', "$this->baseUrl/api/v1/users/{$this->username}/heatmap", [ 'headers' => ['Authorization' => "token {$this->token}"], ]); + + return ['response' => $response, 'since' => $since, 'until' => $until]; } /** - * @return array date (Y-m-d) => contribution count + * Filters the heatmap entries to the [$since, $until] window and sums contributions per day. + * + * @param array{response: ResponseInterface, since: ?\DateTimeImmutable, until: ?\DateTimeImmutable} $handle + * @return array date (Y-m-d) => contribution count */ public function resolveFetch(mixed $handle): array { - /** @var ResponseInterface $handle */ - $data = $handle->toArray(); - $cutoff = (new \DateTimeImmutable('-365 days'))->getTimestamp(); - $result = []; + ['response' => $response, 'since' => $since, 'until' => $until] = $handle; + + /** @var ResponseInterface $response */ + $data = $response->toArray(); + $cutoff = ($since ?? new \DateTimeImmutable('-365 days'))->getTimestamp(); + $ceiling = $until?->getTimestamp(); + $result = []; foreach ($data as $entry) { if ($entry['timestamp'] < $cutoff) { continue; } + if ($ceiling !== null && $entry['timestamp'] > $ceiling) { + continue; + } $date = date('Y-m-d', $entry['timestamp']); $result[$date] = ($result[$date] ?? 0) + (int) $entry['contributions']; } diff --git a/src/Service/Provider/ProviderInterface.php b/src/Service/Provider/ProviderInterface.php index 2daa0fe..dad0613 100644 --- a/src/Service/Provider/ProviderInterface.php +++ b/src/Service/Provider/ProviderInterface.php @@ -6,17 +6,33 @@ namespace GitContributionGraph\Service\Provider; interface ProviderInterface { - /** Fire the HTTP request(s) without blocking; returns an opaque handle for resolveFetch(). */ - public function startFetch(): mixed; + /** + * Fire the HTTP request(s) without blocking; the result is read later by + * resolveFetch() so multiple providers' requests can be in flight at once. + * + * @param ?\DateTimeImmutable $since lower bound (inclusive); defaults to 365 days ago + * @param ?\DateTimeImmutable $until upper bound (inclusive); defaults to now + * @return mixed opaque handle to pass into resolveFetch() + */ + public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): mixed; - /** @return array date (Y-m-d) => contribution count */ + /** + * Block on the handle from startFetch() and parse it into a per-day count map. + * + * @param mixed $handle the value returned by startFetch() + * @return array date (Y-m-d) => contribution count + */ public function resolveFetch(mixed $handle): array; + /** Whether the required credentials/URL for this provider are all set. */ public function isConfigured(): bool; + /** Short lowercase identifier for this provider, e.g. "github". */ public function getName(): string; + /** Check reachability/credentials without fetching contribution data; used by the /health endpoint. */ public function probe(): ProviderStatus; + /** Make a lightweight authenticated request to verify the provider is reachable; throws on failure. */ public function ping(): void; } diff --git a/tests/Unit/Command/RefetchContributionsCommandTest.php b/tests/Unit/Command/RefetchContributionsCommandTest.php new file mode 100644 index 0000000..cbc805d --- /dev/null +++ b/tests/Unit/Command/RefetchContributionsCommandTest.php @@ -0,0 +1,149 @@ + $fresh */ + private function makeProvider( + string $name, + bool $configured = true, + ?array $fresh = null, + ?\Throwable $throws = null, + ): ProviderInterface { + $provider = $this->createStub(ProviderInterface::class); + $provider->method('getName')->willReturn($name); + $provider->method('isConfigured')->willReturn($configured); + + if ($throws !== null) { + $provider->method('startFetch')->willThrowException($throws); + } elseif ($fresh !== null) { + $provider->method('resolveFetch')->willReturn($fresh); + } + + return $provider; + } + + /** @param iterable $providers */ + private function makeTester(iterable $providers, ContributionStore $store): CommandTester + { + $command = new RefetchContributionsCommand($providers, $store); + + $application = new Application(); + $application->add($command); + + return new CommandTester($application->find('graph:contributions:refetch')); + } + + #[Test] + public function it_refetches_the_default_365_day_window_for_all_configured_providers(): void + { + $store = new ContributionStore(':memory:'); + $provider = $this->makeProvider('github', fresh: ['2024-06-10' => 3]); + + $tester = $this->makeTester([$provider], $store); + $exitCode = $tester->execute([]); + + $this->assertSame(0, $exitCode); + $this->assertCount(1, $store->all('github')); + } + + #[Test] + public function it_skips_unconfigured_providers_by_default(): void + { + $store = new ContributionStore(':memory:'); + $provider = $this->makeProvider('github', configured: false); + + $tester = $this->makeTester([$provider], $store); + $tester->execute([]); + + $this->assertCount(0, $store->all('github')); + } + + #[Test] + public function it_fails_cleanly_on_an_unknown_provider_name(): void + { + $store = new ContributionStore(':memory:'); + $provider = $this->makeProvider('github'); + + $tester = $this->makeTester([$provider], $store); + $exitCode = $tester->execute(['--provider' => 'bogus']); + + $this->assertSame(1, $exitCode); + $this->assertStringContainsString('Unknown provider: bogus', $tester->getDisplay()); + } + + #[Test] + public function it_restricts_to_the_named_provider(): void + { + $store = new ContributionStore(':memory:'); + $github = $this->makeProvider('github', fresh: ['2024-06-10' => 1]); + $gitlab = $this->makeProvider('gitlab', fresh: ['2024-06-10' => 1]); + + $tester = $this->makeTester([$github, $gitlab], $store); + $tester->execute(['--provider' => 'github']); + + $this->assertCount(1, $store->all('github')); + $this->assertCount(0, $store->all('gitlab')); + } + + #[Test] + public function it_continues_with_remaining_providers_when_one_throws(): void + { + $store = new ContributionStore(':memory:'); + $failing = $this->makeProvider('gitlab', throws: new \RuntimeException('boom')); + $healthy = $this->makeProvider('github', fresh: ['2024-06-10' => 2]); + + $tester = $this->makeTester([$failing, $healthy], $store); + $exitCode = $tester->execute([]); + + $this->assertSame(0, $exitCode); + $this->assertCount(1, $store->all('github')); + $this->assertCount(0, $store->all('gitlab')); + } + + #[Test] + public function it_fails_when_every_provider_errors(): void + { + $store = new ContributionStore(':memory:'); + $failing = $this->makeProvider('github', throws: new \RuntimeException('boom')); + + $tester = $this->makeTester([$failing], $store); + $exitCode = $tester->execute([]); + + $this->assertSame(1, $exitCode); + } + + #[Test] + public function it_splits_a_multi_year_range_into_365_day_chunks(): void + { + $store = new ContributionStore(':memory:'); + $calls = 0; + + $provider = $this->createStub(ProviderInterface::class); + $provider->method('getName')->willReturn('github'); + $provider->method('isConfigured')->willReturn(true); + $provider->method('resolveFetch')->willReturnCallback(function () use (&$calls): array { + $calls++; + + return []; + }); + + $tester = $this->makeTester([$provider], $store); + $tester->execute(['--from' => '2020-01-01', '--to' => '2023-01-01']); + + $this->assertGreaterThan(1, $calls); + } +} diff --git a/tests/Unit/Service/ContributionAggregatorTest.php b/tests/Unit/Service/ContributionAggregatorTest.php index 5120026..9c939c3 100644 --- a/tests/Unit/Service/ContributionAggregatorTest.php +++ b/tests/Unit/Service/ContributionAggregatorTest.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace GitContributionGraph\Tests\Unit\Service; use GitContributionGraph\Service\ContributionAggregator; +use GitContributionGraph\Service\ContributionStore; use GitContributionGraph\Service\Provider\ProviderInterface; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\Test; @@ -15,16 +16,31 @@ use Psr\Log\LoggerInterface; final class ContributionAggregatorTest extends TestCase { private LoggerInterface $logger; + private ContributionStore $store; protected function setUp(): void { $this->logger = $this->createStub(LoggerInterface::class); + $this->store = new ContributionStore(':memory:'); + } + + /** @param ?array $fresh */ + private function makeProvider(string $name, bool $configured = true, ?array $fresh = null): ProviderInterface + { + $provider = $this->createStub(ProviderInterface::class); + $provider->method('getName')->willReturn($name); + $provider->method('isConfigured')->willReturn($configured); + if ($fresh !== null) { + $provider->method('resolveFetch')->willReturn($fresh); + } + + return $provider; } #[Test] public function it_returns_empty_array_when_no_providers_are_given(): void { - $aggregator = new ContributionAggregator([], $this->logger); + $aggregator = new ContributionAggregator([], $this->store, $this->logger); $result = $aggregator->aggregate(); @@ -34,10 +50,9 @@ final class ContributionAggregatorTest extends TestCase #[Test] public function it_skips_unconfigured_providers(): void { - $provider = $this->createStub(ProviderInterface::class); - $provider->method('isConfigured')->willReturn(false); + $provider = $this->makeProvider('github', configured: false); - $aggregator = new ContributionAggregator([$provider], $this->logger); + $aggregator = new ContributionAggregator([$provider], $this->store, $this->logger); $result = $aggregator->aggregate(); @@ -47,69 +62,68 @@ final class ContributionAggregatorTest extends TestCase #[Test] public function it_returns_contributions_from_a_configured_provider(): void { - $provider = $this->createStub(ProviderInterface::class); - $provider->method('isConfigured')->willReturn(true); - $provider->method('resolveFetch')->willReturn(['2024-01-01' => 3]); + $date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d'); + $provider = $this->makeProvider('github', fresh: [$date => 3]); - $aggregator = new ContributionAggregator([$provider], $this->logger); + $aggregator = new ContributionAggregator([$provider], $this->store, $this->logger); $result = $aggregator->aggregate(); - $this->assertSame(['2024-01-01' => 3], $result); + $this->assertSame([$date => 3], $result); } #[Test] public function it_sums_contributions_from_multiple_providers_on_the_same_date(): void { - $providerA = $this->createStub(ProviderInterface::class); - $providerA->method('isConfigured')->willReturn(true); - $providerA->method('resolveFetch')->willReturn(['2024-01-01' => 3, '2024-01-02' => 1]); + $dateA = (new \DateTimeImmutable('-1 day'))->format('Y-m-d'); + $dateB = (new \DateTimeImmutable('-2 days'))->format('Y-m-d'); - $providerB = $this->createStub(ProviderInterface::class); - $providerB->method('isConfigured')->willReturn(true); - $providerB->method('resolveFetch')->willReturn(['2024-01-01' => 2, '2024-01-03' => 5]); + $providerA = $this->makeProvider('github', fresh: [$dateA => 3, $dateB => 1]); + $providerB = $this->makeProvider('gitlab', fresh: [$dateA => 2]); - $aggregator = new ContributionAggregator([$providerA, $providerB], $this->logger); + $aggregator = new ContributionAggregator([$providerA, $providerB], $this->store, $this->logger); $result = $aggregator->aggregate(); - $this->assertSame(['2024-01-01' => 5, '2024-01-02' => 1, '2024-01-03' => 5], $result); + $this->assertSame([$dateB => 1, $dateA => 5], $result); } #[Test] public function it_continues_fetching_remaining_providers_when_one_throws_on_start(): void { + $date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d'); + $failing = $this->createStub(ProviderInterface::class); + $failing->method('getName')->willReturn('gitlab'); $failing->method('isConfigured')->willReturn(true); $failing->method('startFetch')->willThrowException(new \RuntimeException('Network error')); - $healthy = $this->createStub(ProviderInterface::class); - $healthy->method('isConfigured')->willReturn(true); - $healthy->method('resolveFetch')->willReturn(['2024-01-01' => 7]); + $healthy = $this->makeProvider('github', fresh: [$date => 7]); - $aggregator = new ContributionAggregator([$failing, $healthy], $this->logger); + $aggregator = new ContributionAggregator([$failing, $healthy], $this->store, $this->logger); $result = $aggregator->aggregate(); - $this->assertSame(['2024-01-01' => 7], $result); + $this->assertSame([$date => 7], $result); } #[Test] public function it_continues_fetching_remaining_providers_when_one_throws_on_resolve(): void { + $date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d'); + $failing = $this->createStub(ProviderInterface::class); + $failing->method('getName')->willReturn('gitlab'); $failing->method('isConfigured')->willReturn(true); $failing->method('resolveFetch')->willThrowException(new \RuntimeException('Network error')); - $healthy = $this->createStub(ProviderInterface::class); - $healthy->method('isConfigured')->willReturn(true); - $healthy->method('resolveFetch')->willReturn(['2024-01-01' => 7]); + $healthy = $this->makeProvider('github', fresh: [$date => 7]); - $aggregator = new ContributionAggregator([$failing, $healthy], $this->logger); + $aggregator = new ContributionAggregator([$failing, $healthy], $this->store, $this->logger); $result = $aggregator->aggregate(); - $this->assertSame(['2024-01-01' => 7], $result); + $this->assertSame([$date => 7], $result); } #[Test] @@ -119,10 +133,11 @@ final class ContributionAggregatorTest extends TestCase $logger->expects($this->once())->method('warning'); $provider = $this->createStub(ProviderInterface::class); + $provider->method('getName')->willReturn('github'); $provider->method('isConfigured')->willReturn(true); $provider->method('startFetch')->willThrowException(new \RuntimeException('fail')); - (new ContributionAggregator([$provider], $logger))->aggregate(); + (new ContributionAggregator([$provider], $this->store, $logger))->aggregate(); } #[Test] @@ -132,9 +147,77 @@ final class ContributionAggregatorTest extends TestCase $logger->expects($this->once())->method('warning'); $provider = $this->createStub(ProviderInterface::class); + $provider->method('getName')->willReturn('github'); $provider->method('isConfigured')->willReturn(true); $provider->method('resolveFetch')->willThrowException(new \RuntimeException('fail')); - (new ContributionAggregator([$provider], $logger))->aggregate(); + (new ContributionAggregator([$provider], $this->store, $logger))->aggregate(); + } + + #[Test] + public function it_consults_latest_date_and_narrows_the_since_window_by_three_days(): void + { + $latest = (new \DateTimeImmutable('-30 days'))->getTimestamp(); + $this->store->add('github', $latest, 1); + + $provider = $this->createMock(ProviderInterface::class); + $provider->method('getName')->willReturn('github'); + $provider->method('isConfigured')->willReturn(true); + $provider->expects($this->once()) + ->method('startFetch') + ->with($this->callback( + fn (?\DateTimeImmutable $since): bool => $since !== null + && $since->getTimestamp() === $latest - 3 * 86400 + )) + ->willReturn(null); + $provider->method('resolveFetch')->willReturn([]); + + (new ContributionAggregator([$provider], $this->store, $this->logger))->aggregate(); + } + + #[Test] + public function it_merges_freshly_fetched_contributions_into_the_store(): void + { + $date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d'); + $provider = $this->makeProvider('github', fresh: [$date => 4]); + + (new ContributionAggregator([$provider], $this->store, $this->logger))->aggregate(); + + $stored = iterator_to_array($this->store->all('github')); + $this->assertCount(1, $stored); + $this->assertSame(4, $stored[0]->count); + } + + #[Test] + public function it_prunes_the_store_once_per_aggregate_call(): void + { + $store = $this->createMock(ContributionStore::class); + $store->method('latestDate')->willReturn(null); + $store->method('all')->willReturn(new \GitContributionGraph\Entity\ContributionCollection()); + $store->expects($this->once())->method('prune'); + + $provider = $this->makeProvider('github', fresh: []); + + (new ContributionAggregator([$provider], $store, $this->logger))->aggregate(); + } + + #[Test] + public function it_leaves_other_providers_stored_data_intact_when_one_provider_fails(): void + { + $date = (new \DateTimeImmutable('-1 day'))->getTimestamp(); + $this->store->add('gitlab', $date, 9); + + $failing = $this->createStub(ProviderInterface::class); + $failing->method('getName')->willReturn('gitlab'); + $failing->method('isConfigured')->willReturn(true); + $failing->method('startFetch')->willThrowException(new \RuntimeException('Network error')); + + $healthy = $this->makeProvider('github', fresh: []); + + (new ContributionAggregator([$failing, $healthy], $this->store, $this->logger))->aggregate(); + + $stored = iterator_to_array($this->store->all('gitlab')); + $this->assertCount(1, $stored); + $this->assertSame(9, $stored[0]->count); } } diff --git a/tests/Unit/Service/Provider/GitHubProviderTest.php b/tests/Unit/Service/Provider/GitHubProviderTest.php index a3095cc..da6a6eb 100644 --- a/tests/Unit/Service/Provider/GitHubProviderTest.php +++ b/tests/Unit/Service/Provider/GitHubProviderTest.php @@ -29,6 +29,7 @@ final class GitHubProviderTest extends TestCase ); } + /** @param array}> $weeks */ private function stubGraphqlResponse(array $weeks): ResponseInterface { $response = $this->createStub(ResponseInterface::class); @@ -131,4 +132,27 @@ final class GitHubProviderTest extends TestCase $this->assertSame([], $result); } + + #[Test] + public function it_narrows_the_graphql_query_window_when_since_and_until_are_given(): void + { + $since = new \DateTimeImmutable('2024-01-01'); + $until = new \DateTimeImmutable('2024-01-31'); + $capturedQuery = null; + + $client = $this->createMock(HttpClientInterface::class); + $client->method('request')->willReturnCallback( + function (string $method, string $url, array $options) use (&$capturedQuery): ResponseInterface { + $capturedQuery = $options['json']['query']; + + return $this->stubGraphqlResponse([]); + } + ); + + $provider = $this->makeProvider(client: $client); + $provider->startFetch($since, $until); + + $this->assertStringContainsString('2024-01-01T00:00:00Z', $capturedQuery); + $this->assertStringContainsString('2024-01-31T23:59:59Z', $capturedQuery); + } } diff --git a/tests/Unit/Service/Provider/GitLabProviderTest.php b/tests/Unit/Service/Provider/GitLabProviderTest.php index 34fbd9b..5673318 100644 --- a/tests/Unit/Service/Provider/GitLabProviderTest.php +++ b/tests/Unit/Service/Provider/GitLabProviderTest.php @@ -31,6 +31,7 @@ final class GitLabProviderTest extends TestCase ); } + /** @param array> $data raw JSON body, e.g. events or the user lookup */ private function stubResponse(array $data): ResponseInterface { $response = $this->createStub(ResponseInterface::class); @@ -128,4 +129,27 @@ final class GitLabProviderTest extends TestCase $this->assertSame(100, $result['2024-06-10']); $this->assertSame(1, $result['2024-06-11']); } + + #[Test] + public function it_sends_after_and_before_query_params_when_since_and_until_are_given(): void + { + $capturedQuery = null; + + $client = $this->createMock(HttpClientInterface::class); + $client->method('request')->willReturnCallback( + function (string $method, string $url, array $options = []) use (&$capturedQuery): ResponseInterface { + if (str_contains($url, '/events')) { + $capturedQuery = $options['query']; + } + + return $this->stubResponse(str_contains($url, '/events') ? [] : [['id' => 42]]); + } + ); + + $provider = $this->makeProvider(client: $client); + $provider->startFetch(new \DateTimeImmutable('2024-01-01'), new \DateTimeImmutable('2024-01-31')); + + $this->assertSame('2024-01-01', $capturedQuery['after']); + $this->assertSame('2024-01-31', $capturedQuery['before']); + } } diff --git a/tests/Unit/Service/Provider/GiteaProviderTest.php b/tests/Unit/Service/Provider/GiteaProviderTest.php index ae12471..3dd8b6d 100644 --- a/tests/Unit/Service/Provider/GiteaProviderTest.php +++ b/tests/Unit/Service/Provider/GiteaProviderTest.php @@ -30,6 +30,7 @@ final class GiteaProviderTest extends TestCase ); } + /** @param array $data */ private function stubResponse(array $data): ResponseInterface { $response = $this->createStub(ResponseInterface::class); @@ -111,4 +112,44 @@ final class GiteaProviderTest extends TestCase $this->assertSame([], $result); } + + #[Test] + public function it_filters_out_entries_before_since(): void + { + $since = new \DateTimeImmutable('2024-01-10'); + $before = $since->modify('-1 day')->getTimestamp(); + $after = $since->modify('+1 day')->getTimestamp(); + + $client = $this->createStub(HttpClientInterface::class); + $client->method('request')->willReturn($this->stubResponse([ + ['timestamp' => $before, 'contributions' => 3], + ['timestamp' => $after, 'contributions' => 2], + ])); + + $provider = $this->makeProvider(client: $client); + $result = $provider->resolveFetch($provider->startFetch($since)); + + $this->assertArrayNotHasKey(date('Y-m-d', $before), $result); + $this->assertSame(2, $result[date('Y-m-d', $after)]); + } + + #[Test] + public function it_filters_out_entries_after_until(): void + { + $until = new \DateTimeImmutable('-10 days'); + $before = $until->modify('-1 day')->getTimestamp(); + $after = $until->modify('+1 day')->getTimestamp(); + + $client = $this->createStub(HttpClientInterface::class); + $client->method('request')->willReturn($this->stubResponse([ + ['timestamp' => $before, 'contributions' => 3], + ['timestamp' => $after, 'contributions' => 2], + ])); + + $provider = $this->makeProvider(client: $client); + $result = $provider->resolveFetch($provider->startFetch(null, $until)); + + $this->assertSame(3, $result[date('Y-m-d', $before)]); + $this->assertArrayNotHasKey(date('Y-m-d', $after), $result); + } } diff --git a/tests/Unit/Service/Provider/ProbeTraitTest.php b/tests/Unit/Service/Provider/ProbeTraitTest.php index bd6ba28..d680d80 100644 --- a/tests/Unit/Service/Provider/ProbeTraitTest.php +++ b/tests/Unit/Service/Provider/ProbeTraitTest.php @@ -31,7 +31,7 @@ final class ProbeTraitTest extends TestCase public function isConfigured(): bool { return $this->configured; } public function getName(): string { return 'test'; } public function ping(): void { ($this->ping)(); } - public function startFetch(): mixed { return null; } + public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): mixed { return null; } public function resolveFetch(mixed $handle): array { return []; } }; }