From a234c2c992835bd74abdd6dbd8022250e5002ead Mon Sep 17 00:00:00 2001 From: Melody Horn Date: Tue, 2 Mar 2021 23:25:03 -0700 Subject: don't make assumptions about tarball order --- update.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/update.py b/update.py index 64a7768..7f84193 100644 --- a/update.py +++ b/update.py @@ -1,3 +1,4 @@ +from collections import defaultdict import csv from dataclasses import dataclass import datetime @@ -83,7 +84,7 @@ if not dump_tarball.exists(): csv.field_size_limit(69696969) dump = tarfile.open(dump_tarball) -crates = dict() +crates = defaultdict(lambda: Crate('', -1)) metadata = None for item in dump: if item.name.endswith('metadata.json'): @@ -91,9 +92,9 @@ for item in dump: elif item.name.endswith('crates.csv'): reader = csv.DictReader(io.TextIOWrapper(dump.extractfile(item), 'UTF-8')) for crate in reader: - crates[crate['id']] = Crate(crate['name'], int(crate['downloads'])) + crates[crate['id']].name = crate['name'] + crates[crate['id']].downloads = int(crate['downloads']) elif item.name.endswith('versions.csv'): - assert len(crates) > 0, "versions read before crates!" reader = csv.DictReader(io.TextIOWrapper(dump.extractfile(item), 'UTF-8')) for version in reader: if version['yanked'] == 't': -- cgit v1.2.3