haylan/git-contribution-graph
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

feat(store): fetch contributions incrementally and persist history in SQLite

Browse Source 
Bound each provider's startFetch()/resolveFetch() to an optional
since/until window, wire a SQLite-backed ContributionStore into
ContributionAggregator (fetch only the trailing window past the last
stored date, merge into the store, prune by CONTRIBUTIONS_RETENTION_DAYS),
and add a graph:contributions:refetch command to force a full or ranged
re-fetch in <=365-day chunks. This is the root fix for the full
365-day-refetch timeout that used to hit on every cache miss.
This commit is contained in:
haylan
2026-07-12 23:42:41 +02:00
parent 41e88144a4
commit 4ff45b7c46
16 changed files with 642 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Unit/Command/RefetchContributionsCommandTest.php
+149
View File
@@ -0,0 +1,149 @@
<?php
declare(strict_types=1);
namespace GitContributionGraph\Tests\Unit\Command;
use GitContributionGraph\Command\RefetchContributionsCommand;
use GitContributionGraph\Service\ContributionStore;
use GitContributionGraph\Service\Provider\ProviderInterface;
use PHPUnit\Framework\Attributes\CoversClass;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
use Symfony\Component\Console\Application;
use Symfony\Component\Console\Tester\CommandTester;
#[CoversClass(RefetchContributionsCommand::class)]
final class RefetchContributionsCommandTest extends TestCase
{
/** @param ?array<string, int> $fresh */
private function makeProvider(
string $name,
bool $configured = true,
?array $fresh = null,
?\Throwable $throws = null,
): ProviderInterface {
$provider = $this->createStub(ProviderInterface::class);
$provider->method('getName')->willReturn($name);
$provider->method('isConfigured')->willReturn($configured);
if ($throws !== null) {
$provider->method('startFetch')->willThrowException($throws);
} elseif ($fresh !== null) {
$provider->method('resolveFetch')->willReturn($fresh);
}
return $provider;
}
/** @param iterable<ProviderInterface> $providers */
private function makeTester(iterable $providers, ContributionStore $store): CommandTester
{
$command = new RefetchContributionsCommand($providers, $store);
$application = new Application();
$application->add($command);
return new CommandTester($application->find('graph:contributions:refetch'));
}
#[Test]
public function it_refetches_the_default_365_day_window_for_all_configured_providers(): void
{
$store = new ContributionStore(':memory:');
$provider = $this->makeProvider('github', fresh: ['2024-06-10' => 3]);
$tester = $this->makeTester([$provider], $store);
$exitCode = $tester->execute([]);
$this->assertSame(0, $exitCode);
$this->assertCount(1, $store->all('github'));
}
#[Test]
public function it_skips_unconfigured_providers_by_default(): void
{
$store = new ContributionStore(':memory:');
$provider = $this->makeProvider('github', configured: false);
$tester = $this->makeTester([$provider], $store);
$tester->execute([]);
$this->assertCount(0, $store->all('github'));
}
#[Test]
public function it_fails_cleanly_on_an_unknown_provider_name(): void
{
$store = new ContributionStore(':memory:');
$provider = $this->makeProvider('github');
$tester = $this->makeTester([$provider], $store);
$exitCode = $tester->execute(['--provider' => 'bogus']);
$this->assertSame(1, $exitCode);
$this->assertStringContainsString('Unknown provider: bogus', $tester->getDisplay());
}
#[Test]
public function it_restricts_to_the_named_provider(): void
{
$store = new ContributionStore(':memory:');
$github = $this->makeProvider('github', fresh: ['2024-06-10' => 1]);
$gitlab = $this->makeProvider('gitlab', fresh: ['2024-06-10' => 1]);
$tester = $this->makeTester([$github, $gitlab], $store);
$tester->execute(['--provider' => 'github']);
$this->assertCount(1, $store->all('github'));
$this->assertCount(0, $store->all('gitlab'));
}
#[Test]
public function it_continues_with_remaining_providers_when_one_throws(): void
{
$store = new ContributionStore(':memory:');
$failing = $this->makeProvider('gitlab', throws: new \RuntimeException('boom'));
$healthy = $this->makeProvider('github', fresh: ['2024-06-10' => 2]);
$tester = $this->makeTester([$failing, $healthy], $store);
$exitCode = $tester->execute([]);
$this->assertSame(0, $exitCode);
$this->assertCount(1, $store->all('github'));
$this->assertCount(0, $store->all('gitlab'));
}
#[Test]
public function it_fails_when_every_provider_errors(): void
{
$store = new ContributionStore(':memory:');
$failing = $this->makeProvider('github', throws: new \RuntimeException('boom'));
$tester = $this->makeTester([$failing], $store);
$exitCode = $tester->execute([]);
$this->assertSame(1, $exitCode);
}
#[Test]
public function it_splits_a_multi_year_range_into_365_day_chunks(): void
{
$store = new ContributionStore(':memory:');
$calls = 0;
$provider = $this->createStub(ProviderInterface::class);
$provider->method('getName')->willReturn('github');
$provider->method('isConfigured')->willReturn(true);
$provider->method('resolveFetch')->willReturnCallback(function () use (&$calls): array {
$calls++;
return [];
});
$tester = $this->makeTester([$provider], $store);
$tester->execute(['--from' => '2020-01-01', '--to' => '2023-01-01']);
$this->assertGreaterThan(1, $calls);
}
}
tests/Unit/Service/ContributionAggregatorTest.php
+112 -29
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace GitContributionGraph\Tests\Unit\Service;
use GitContributionGraph\Service\ContributionAggregator;
use GitContributionGraph\Service\ContributionStore;
use GitContributionGraph\Service\Provider\ProviderInterface;
use PHPUnit\Framework\Attributes\CoversClass;
use PHPUnit\Framework\Attributes\Test;
@@ -15,16 +16,31 @@ use Psr\Log\LoggerInterface;
final class ContributionAggregatorTest extends TestCase
{
private LoggerInterface $logger;
private ContributionStore $store;
protected function setUp(): void
{
$this->logger = $this->createStub(LoggerInterface::class);
$this->store = new ContributionStore(':memory:');
}
/** @param ?array<string, int> $fresh */
private function makeProvider(string $name, bool $configured = true, ?array $fresh = null): ProviderInterface
{
$provider = $this->createStub(ProviderInterface::class);
$provider->method('getName')->willReturn($name);
$provider->method('isConfigured')->willReturn($configured);
if ($fresh !== null) {
$provider->method('resolveFetch')->willReturn($fresh);
}
return $provider;
}
#[Test]
public function it_returns_empty_array_when_no_providers_are_given(): void
{
$aggregator = new ContributionAggregator([], $this->logger);
$aggregator = new ContributionAggregator([], $this->store, $this->logger);
$result = $aggregator->aggregate();
@@ -34,10 +50,9 @@ final class ContributionAggregatorTest extends TestCase
#[Test]
public function it_skips_unconfigured_providers(): void
{
$provider = $this->createStub(ProviderInterface::class);
$provider->method('isConfigured')->willReturn(false);
$provider = $this->makeProvider('github', configured: false);
$aggregator = new ContributionAggregator([$provider], $this->logger);
$aggregator = new ContributionAggregator([$provider], $this->store, $this->logger);
$result = $aggregator->aggregate();
@@ -47,69 +62,68 @@ final class ContributionAggregatorTest extends TestCase
#[Test]
public function it_returns_contributions_from_a_configured_provider(): void
{
$provider = $this->createStub(ProviderInterface::class);
$provider->method('isConfigured')->willReturn(true);
$provider->method('resolveFetch')->willReturn(['2024-01-01' => 3]);
$date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d');
$provider = $this->makeProvider('github', fresh: [$date => 3]);
$aggregator = new ContributionAggregator([$provider], $this->logger);
$aggregator = new ContributionAggregator([$provider], $this->store, $this->logger);
$result = $aggregator->aggregate();
$this->assertSame(['2024-01-01' => 3], $result);
$this->assertSame([$date => 3], $result);
}
#[Test]
public function it_sums_contributions_from_multiple_providers_on_the_same_date(): void
{
$providerA = $this->createStub(ProviderInterface::class);
$providerA->method('isConfigured')->willReturn(true);
$providerA->method('resolveFetch')->willReturn(['2024-01-01' => 3, '2024-01-02' => 1]);
$dateA = (new \DateTimeImmutable('-1 day'))->format('Y-m-d');
$dateB = (new \DateTimeImmutable('-2 days'))->format('Y-m-d');
$providerB = $this->createStub(ProviderInterface::class);
$providerB->method('isConfigured')->willReturn(true);
$providerB->method('resolveFetch')->willReturn(['2024-01-01' => 2, '2024-01-03' => 5]);
$providerA = $this->makeProvider('github', fresh: [$dateA => 3, $dateB => 1]);
$providerB = $this->makeProvider('gitlab', fresh: [$dateA => 2]);
$aggregator = new ContributionAggregator([$providerA, $providerB], $this->logger);
$aggregator = new ContributionAggregator([$providerA, $providerB], $this->store, $this->logger);
$result = $aggregator->aggregate();
$this->assertSame(['2024-01-01' => 5, '2024-01-02' => 1, '2024-01-03' => 5], $result);
$this->assertSame([$dateB => 1, $dateA => 5], $result);
}
#[Test]
public function it_continues_fetching_remaining_providers_when_one_throws_on_start(): void
{
$date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d');
$failing = $this->createStub(ProviderInterface::class);
$failing->method('getName')->willReturn('gitlab');
$failing->method('isConfigured')->willReturn(true);
$failing->method('startFetch')->willThrowException(new \RuntimeException('Network error'));
$healthy = $this->createStub(ProviderInterface::class);
$healthy->method('isConfigured')->willReturn(true);
$healthy->method('resolveFetch')->willReturn(['2024-01-01' => 7]);
$healthy = $this->makeProvider('github', fresh: [$date => 7]);
$aggregator = new ContributionAggregator([$failing, $healthy], $this->logger);
$aggregator = new ContributionAggregator([$failing, $healthy], $this->store, $this->logger);
$result = $aggregator->aggregate();
$this->assertSame(['2024-01-01' => 7], $result);
$this->assertSame([$date => 7], $result);
}
#[Test]
public function it_continues_fetching_remaining_providers_when_one_throws_on_resolve(): void
{
$date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d');
$failing = $this->createStub(ProviderInterface::class);
$failing->method('getName')->willReturn('gitlab');
$failing->method('isConfigured')->willReturn(true);
$failing->method('resolveFetch')->willThrowException(new \RuntimeException('Network error'));
$healthy = $this->createStub(ProviderInterface::class);
$healthy->method('isConfigured')->willReturn(true);
$healthy->method('resolveFetch')->willReturn(['2024-01-01' => 7]);
$healthy = $this->makeProvider('github', fresh: [$date => 7]);
$aggregator = new ContributionAggregator([$failing, $healthy], $this->logger);
$aggregator = new ContributionAggregator([$failing, $healthy], $this->store, $this->logger);
$result = $aggregator->aggregate();
$this->assertSame(['2024-01-01' => 7], $result);
$this->assertSame([$date => 7], $result);
}
#[Test]
@@ -119,10 +133,11 @@ final class ContributionAggregatorTest extends TestCase
$logger->expects($this->once())->method('warning');
$provider = $this->createStub(ProviderInterface::class);
$provider->method('getName')->willReturn('github');
$provider->method('isConfigured')->willReturn(true);
$provider->method('startFetch')->willThrowException(new \RuntimeException('fail'));
(new ContributionAggregator([$provider], $logger))->aggregate();
(new ContributionAggregator([$provider], $this->store, $logger))->aggregate();
}
#[Test]
@@ -132,9 +147,77 @@ final class ContributionAggregatorTest extends TestCase
$logger->expects($this->once())->method('warning');
$provider = $this->createStub(ProviderInterface::class);
$provider->method('getName')->willReturn('github');
$provider->method('isConfigured')->willReturn(true);
$provider->method('resolveFetch')->willThrowException(new \RuntimeException('fail'));
(new ContributionAggregator([$provider], $logger))->aggregate();
(new ContributionAggregator([$provider], $this->store, $logger))->aggregate();
}
#[Test]
public function it_consults_latest_date_and_narrows_the_since_window_by_three_days(): void
{
$latest = (new \DateTimeImmutable('-30 days'))->getTimestamp();
$this->store->add('github', $latest, 1);
$provider = $this->createMock(ProviderInterface::class);
$provider->method('getName')->willReturn('github');
$provider->method('isConfigured')->willReturn(true);
$provider->expects($this->once())
->method('startFetch')
->with($this->callback(
fn (?\DateTimeImmutable $since): bool => $since !== null
&& $since->getTimestamp() === $latest - 3 * 86400
))
->willReturn(null);
$provider->method('resolveFetch')->willReturn([]);
(new ContributionAggregator([$provider], $this->store, $this->logger))->aggregate();
}
#[Test]
public function it_merges_freshly_fetched_contributions_into_the_store(): void
{
$date = (new \DateTimeImmutable('-1 day'))->format('Y-m-d');
$provider = $this->makeProvider('github', fresh: [$date => 4]);
(new ContributionAggregator([$provider], $this->store, $this->logger))->aggregate();
$stored = iterator_to_array($this->store->all('github'));
$this->assertCount(1, $stored);
$this->assertSame(4, $stored[0]->count);
}
#[Test]
public function it_prunes_the_store_once_per_aggregate_call(): void
{
$store = $this->createMock(ContributionStore::class);
$store->method('latestDate')->willReturn(null);
$store->method('all')->willReturn(new \GitContributionGraph\Entity\ContributionCollection());
$store->expects($this->once())->method('prune');
$provider = $this->makeProvider('github', fresh: []);
(new ContributionAggregator([$provider], $store, $this->logger))->aggregate();
}
#[Test]
public function it_leaves_other_providers_stored_data_intact_when_one_provider_fails(): void
{
$date = (new \DateTimeImmutable('-1 day'))->getTimestamp();
$this->store->add('gitlab', $date, 9);
$failing = $this->createStub(ProviderInterface::class);
$failing->method('getName')->willReturn('gitlab');
$failing->method('isConfigured')->willReturn(true);
$failing->method('startFetch')->willThrowException(new \RuntimeException('Network error'));
$healthy = $this->makeProvider('github', fresh: []);
(new ContributionAggregator([$failing, $healthy], $this->store, $this->logger))->aggregate();
$stored = iterator_to_array($this->store->all('gitlab'));
$this->assertCount(1, $stored);
$this->assertSame(9, $stored[0]->count);
}
}
tests/Unit/Service/Provider/GitHubProviderTest.php
+24
View File
@@ -29,6 +29,7 @@ final class GitHubProviderTest extends TestCase
);
}
/** @param array<int, array{contributionDays: array<int, array{date: string, contributionCount: int}>}> $weeks */
private function stubGraphqlResponse(array $weeks): ResponseInterface
{
$response = $this->createStub(ResponseInterface::class);
@@ -131,4 +132,27 @@ final class GitHubProviderTest extends TestCase
$this->assertSame([], $result);
}
#[Test]
public function it_narrows_the_graphql_query_window_when_since_and_until_are_given(): void
{
$since = new \DateTimeImmutable('2024-01-01');
$until = new \DateTimeImmutable('2024-01-31');
$capturedQuery = null;
$client = $this->createMock(HttpClientInterface::class);
$client->method('request')->willReturnCallback(
function (string $method, string $url, array $options) use (&$capturedQuery): ResponseInterface {
$capturedQuery = $options['json']['query'];
return $this->stubGraphqlResponse([]);
}
);
$provider = $this->makeProvider(client: $client);
$provider->startFetch($since, $until);
$this->assertStringContainsString('2024-01-01T00:00:00Z', $capturedQuery);
$this->assertStringContainsString('2024-01-31T23:59:59Z', $capturedQuery);
}
}
tests/Unit/Service/Provider/GitLabProviderTest.php
+24
View File
@@ -31,6 +31,7 @@ final class GitLabProviderTest extends TestCase
);
}
/** @param array<int, array<string, int|string>> $data raw JSON body, e.g. events or the user lookup */
private function stubResponse(array $data): ResponseInterface
{
$response = $this->createStub(ResponseInterface::class);
@@ -128,4 +129,27 @@ final class GitLabProviderTest extends TestCase
$this->assertSame(100, $result['2024-06-10']);
$this->assertSame(1, $result['2024-06-11']);
}
#[Test]
public function it_sends_after_and_before_query_params_when_since_and_until_are_given(): void
{
$capturedQuery = null;
$client = $this->createMock(HttpClientInterface::class);
$client->method('request')->willReturnCallback(
function (string $method, string $url, array $options = []) use (&$capturedQuery): ResponseInterface {
if (str_contains($url, '/events')) {
$capturedQuery = $options['query'];
}
return $this->stubResponse(str_contains($url, '/events') ? [] : [['id' => 42]]);
}
);
$provider = $this->makeProvider(client: $client);
$provider->startFetch(new \DateTimeImmutable('2024-01-01'), new \DateTimeImmutable('2024-01-31'));
$this->assertSame('2024-01-01', $capturedQuery['after']);
$this->assertSame('2024-01-31', $capturedQuery['before']);
}
}
tests/Unit/Service/Provider/GiteaProviderTest.php
+41
View File
@@ -30,6 +30,7 @@ final class GiteaProviderTest extends TestCase
);
}
/** @param array<int, array{timestamp: int, contributions: int}> $data */
private function stubResponse(array $data): ResponseInterface
{
$response = $this->createStub(ResponseInterface::class);
@@ -111,4 +112,44 @@ final class GiteaProviderTest extends TestCase
$this->assertSame([], $result);
}
#[Test]
public function it_filters_out_entries_before_since(): void
{
$since = new \DateTimeImmutable('2024-01-10');
$before = $since->modify('-1 day')->getTimestamp();
$after = $since->modify('+1 day')->getTimestamp();
$client = $this->createStub(HttpClientInterface::class);
$client->method('request')->willReturn($this->stubResponse([
['timestamp' => $before, 'contributions' => 3],
['timestamp' => $after, 'contributions' => 2],
]));
$provider = $this->makeProvider(client: $client);
$result = $provider->resolveFetch($provider->startFetch($since));
$this->assertArrayNotHasKey(date('Y-m-d', $before), $result);
$this->assertSame(2, $result[date('Y-m-d', $after)]);
}
#[Test]
public function it_filters_out_entries_after_until(): void
{
$until = new \DateTimeImmutable('-10 days');
$before = $until->modify('-1 day')->getTimestamp();
$after = $until->modify('+1 day')->getTimestamp();
$client = $this->createStub(HttpClientInterface::class);
$client->method('request')->willReturn($this->stubResponse([
['timestamp' => $before, 'contributions' => 3],
['timestamp' => $after, 'contributions' => 2],
]));
$provider = $this->makeProvider(client: $client);
$result = $provider->resolveFetch($provider->startFetch(null, $until));
$this->assertSame(3, $result[date('Y-m-d', $before)]);
$this->assertArrayNotHasKey(date('Y-m-d', $after), $result);
}
}
tests/Unit/Service/Provider/ProbeTraitTest.php
+1 -1
View File
@@ -31,7 +31,7 @@ final class ProbeTraitTest extends TestCase
public function isConfigured(): bool { return $this->configured; }
public function getName(): string { return 'test'; }
public function ping(): void { ($this->ping)(); }
public function startFetch(): mixed { return null; }
public function startFetch(?\DateTimeImmutable $since = null, ?\DateTimeImmutable $until = null): mixed { return null; }
public function resolveFetch(mixed $handle): array { return []; }
};
}