From e7b83c7c57483277fa243dd7ff1068c8ff33438f Mon Sep 17 00:00:00 2001 From: Melody Horn Date: Mon, 29 Mar 2021 18:27:53 -0600 Subject: fix Last-Modified bc apparently that was broken --- repos/base.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'repos') diff --git a/repos/base.py b/repos/base.py index 220a30f..1bd4ae3 100644 --- a/repos/base.py +++ b/repos/base.py @@ -1,9 +1,7 @@ from dataclasses import dataclass, asdict as dataclass_asdict -import datetime from functools import total_ordering import gzip import json -import os from pathlib import Path import re from typing import Any, Callable, Mapping @@ -73,30 +71,35 @@ class Repository: def get_versions(self) -> Mapping[str, Version]: self._cache_dir().mkdir(parents=True, exist_ok=True) + headers = dict() + downloaded_file = self._cache_file('downloaded') - if downloaded_file.exists(): - mtime = downloaded_file.stat().st_mtime - else: - mtime = 0 - mtime = datetime.datetime.fromtimestamp(mtime, datetime.timezone.utc) - mtime = mtime.strftime(HTTP_DATE) - parsed_file = self._cache_file('parsed.json.gz') + mtime_file = self._cache_file('last-modified') + if mtime_file.exists(): + mtime = mtime_file.read_text() + headers['If-Modified-Since'] = mtime - response = requests.get(self.index_url, headers={ - 'If-Modified-Since': mtime, - }, stream=True) + etag_file = self._cache_file('etag') + if etag_file.exists(): + etag = etag_file.read_text() + headers['If-None-Match'] = etag + + response = requests.get(self.index_url, headers=headers, stream=True) if response.status_code != requests.codes.not_modified: response.raise_for_status() print('Re-downloading', self._full_name()) - set_mtime = response.headers.get('Last-Modified', '') with downloaded_file.open('wb') as f: for chunk in response.iter_content(chunk_size=256): f.write(chunk) - if len(set_mtime) > 0: - set_mtime = datetime.datetime.strptime(set_mtime, HTTP_DATE) - os.utime(downloaded_file, (datetime.datetime.now().timestamp(), set_mtime.timestamp())) + if 'Last-Modified' in response.headers: + set_mtime = response.headers['Last-Modified'] + mtime_file.write_text(set_mtime) + if 'ETag' in response.headers: + set_etag = response.headers['ETag'] + etag_file.write_text(set_etag) + parsed_file = self._cache_file('parsed.json.gz') if response.status_code != requests.codes.not_modified or not parsed_file.exists(): parsed_data = self.parse(downloaded_file) with gzip.open(parsed_file, 'wt') as f: -- cgit v1.2.3