feat(store): fetch contributions incrementally and persist history in SQLite
Bound each provider's startFetch()/resolveFetch() to an optional since/until window, wire a SQLite-backed ContributionStore into ContributionAggregator (fetch only the trailing window past the last stored date, merge into the store, prune by CONTRIBUTIONS_RETENTION_DAYS), and add a graph:contributions:refetch command to force a full or ranged re-fetch in <=365-day chunks. This is the root fix for the full 365-day-refetch timeout that used to hit on every cache miss.
This commit is contained in:
@@ -4,51 +4,188 @@ declare(strict_types=1);
|
||||
|
||||
namespace GitContributionGraph\Command;
|
||||
|
||||
use GitContributionGraph\Service\ContributionStore;
|
||||
use GitContributionGraph\Service\Provider\ProviderInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Attribute\Option;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\ProgressBar;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
use Symfony\Component\DependencyInjection\Attribute\AutowireIterator;
|
||||
|
||||
#[AsCommand(name: 'graph:contributions:refetch')]
|
||||
class RefetchContributionsCommand extends Command
|
||||
#[AsCommand(
|
||||
name: 'graph:contributions:refetch',
|
||||
description: 'Force a full or ranged re-fetch of contribution history into the store.',
|
||||
)]
|
||||
final class RefetchContributionsCommand extends Command
|
||||
{
|
||||
private const MAX_CHUNK_DAYS = 365;
|
||||
private const ALL_SINCE = '2005-01-01';
|
||||
|
||||
private SymfonyStyle $io;
|
||||
|
||||
/** @param iterable<ProviderInterface> $providers */
|
||||
public function __construct(
|
||||
#[AutowireIterator('app.provider')]
|
||||
private readonly iterable $providers,
|
||||
private readonly ContributionStore $store,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
|
||||
public function initialize(InputInterface $input, OutputInterface $output)
|
||||
/** Symfony console lifecycle hook: sets up the styled I/O helper used throughout the command. */
|
||||
public function initialize(InputInterface $input, OutputInterface $output): void
|
||||
{
|
||||
$this->io = new SymfonyStyle($input, $output);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bypasses the aggregator's incremental trailing-window fetch to re-pull a full or explicit
|
||||
* date range for one, several, or all configured providers, chunked into ≤365-day windows.
|
||||
*
|
||||
* @param string $provider comma-separated provider names to refetch; empty = all configured
|
||||
* @param string $from start date (YYYY-MM-DD); empty defaults to 365 days ago
|
||||
* @param string $to end date (YYYY-MM-DD); empty defaults to today
|
||||
* @param bool $all shorthand for --from=2005-01-01
|
||||
* @return int Command::SUCCESS if at least one provider refetched cleanly, else Command::FAILURE
|
||||
*/
|
||||
public function __invoke(
|
||||
//TODO: check if its possible to add the preview with real strings from the providers.
|
||||
#[Option("(repeatable/comma-split, restricts to named provider(s), default = all configured", "provider", "p", "")]
|
||||
string $providers = ""
|
||||
) {
|
||||
$requested = $providers === '' ? null : explode(',', $providers);
|
||||
$byName = [];
|
||||
|
||||
foreach ($this->providers as $provider) {
|
||||
$byName[$provider->getName()] = $provider;
|
||||
#[Option('Comma-separated provider names to refetch (default: all configured)', 'provider', 'p')]
|
||||
string $provider = '',
|
||||
#[Option('Start date (YYYY-MM-DD), default: 365 days ago', 'from')]
|
||||
string $from = '',
|
||||
#[Option('End date (YYYY-MM-DD), default: today', 'to')]
|
||||
string $to = '',
|
||||
#[Option('Refetch full history (from ' . self::ALL_SINCE . ')', 'all')]
|
||||
bool $all = false,
|
||||
): int {
|
||||
$byName = $this->resolveProviders($provider);
|
||||
if ($byName === null) {
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
if ($requested !== null) {
|
||||
foreach ($requested as $name) {
|
||||
if (!isset($byName[$name])) {
|
||||
$this->io->error("Unknown provider: {$name}");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
$fromSpec = match (true) {
|
||||
$all => self::ALL_SINCE,
|
||||
$from !== '' => $from,
|
||||
default => '-365 days',
|
||||
};
|
||||
|
||||
$fromDate = new \DateTimeImmutable($fromSpec);
|
||||
$toDate = new \DateTimeImmutable($to !== '' ? $to : 'today');
|
||||
$chunks = $this->chunk($fromDate, $toDate);
|
||||
|
||||
$summary = [];
|
||||
$succeeded = 0;
|
||||
|
||||
foreach ($byName as $name => $providerInstance) {
|
||||
[$daysWritten, $errors] = $this->refetchProvider($name, $providerInstance, $chunks);
|
||||
|
||||
if ($errors === 0) {
|
||||
$succeeded++;
|
||||
}
|
||||
$byName = array_intersect_key($byName, array_flip($requested));
|
||||
|
||||
$summary[] = [$name, $daysWritten, count($chunks), $errors];
|
||||
}
|
||||
|
||||
$this->io->table(['Provider', 'Days written', 'Chunks fetched', 'Errors'], $summary);
|
||||
|
||||
if ($succeeded === 0) {
|
||||
$this->io->error('No provider completed successfully.');
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$this->io->success(sprintf('%d of %d provider(s) refetched successfully.', $succeeded, count($byName)));
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{0: \DateTimeImmutable, 1: \DateTimeImmutable}> $chunks
|
||||
*
|
||||
* @return array{0: int, 1: int} days written, errors
|
||||
*/
|
||||
private function refetchProvider(string $name, ProviderInterface $provider, array $chunks): array
|
||||
{
|
||||
$this->io->section("Refetching {$name}");
|
||||
|
||||
$progress = new ProgressBar($this->io, count($chunks));
|
||||
$progress->start();
|
||||
|
||||
$daysWritten = 0;
|
||||
$errors = 0;
|
||||
|
||||
foreach ($chunks as [$chunkFrom, $chunkTo]) {
|
||||
try {
|
||||
$fresh = $provider->resolveFetch($provider->startFetch($chunkFrom, $chunkTo));
|
||||
|
||||
$dateCounts = [];
|
||||
foreach ($fresh as $date => $count) {
|
||||
$dateCounts[(new \DateTimeImmutable($date))->getTimestamp()] = $count;
|
||||
}
|
||||
|
||||
$this->store->merge($name, $dateCounts);
|
||||
$daysWritten += count($dateCounts);
|
||||
} catch (\Throwable $e) {
|
||||
$errors++;
|
||||
$this->io->warning("{$name}: {$e->getMessage()}");
|
||||
}
|
||||
|
||||
$progress->advance();
|
||||
}
|
||||
|
||||
$progress->finish();
|
||||
$this->io->newLine(2);
|
||||
|
||||
return [$daysWritten, $errors];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, ProviderInterface>|null null when an unknown provider name was requested
|
||||
*/
|
||||
private function resolveProviders(string $requested): ?array
|
||||
{
|
||||
$byName = [];
|
||||
foreach ($this->providers as $providerInstance) {
|
||||
$byName[$providerInstance->getName()] = $providerInstance;
|
||||
}
|
||||
|
||||
if ($requested === '') {
|
||||
return array_filter($byName, static fn (ProviderInterface $p): bool => $p->isConfigured());
|
||||
}
|
||||
|
||||
$names = explode(',', $requested);
|
||||
foreach ($names as $name) {
|
||||
if (!isset($byName[$name])) {
|
||||
$this->io->error("Unknown provider: {$name}");
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return array_intersect_key($byName, array_flip($names));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<array{0: \DateTimeImmutable, 1: \DateTimeImmutable}>
|
||||
*/
|
||||
private function chunk(\DateTimeImmutable $from, \DateTimeImmutable $to): array
|
||||
{
|
||||
$chunks = [];
|
||||
$cursor = $from;
|
||||
|
||||
while ($cursor < $to) {
|
||||
$chunkEnd = min($cursor->modify('+' . (self::MAX_CHUNK_DAYS - 1) . ' days'), $to);
|
||||
$chunks[] = [$cursor, $chunkEnd];
|
||||
$cursor = $chunkEnd->modify('+1 day');
|
||||
}
|
||||
|
||||
if ($chunks === []) {
|
||||
$chunks[] = [$from, $to];
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user