feeds/update.py: consider https AND http as a fallback
This commit is contained in:
parent
8ae4be341a
commit
3c6da51f84
|
@ -28,14 +28,13 @@
|
|||
''
|
||||
sources_dir=modules/data/feeds/sources
|
||||
name="$1"
|
||||
url="https://$name"
|
||||
json_path="$sources_dir/$name/default.json"
|
||||
|
||||
# the name could have slashes in it, so we want to mkdir -p that
|
||||
# but in a way where the least could go wrong.
|
||||
pushd "$sources_dir"; mkdir -p "$name"; popd
|
||||
|
||||
${./update.py} "$url" "$json_path"
|
||||
${./update.py} "$name" "$json_path"
|
||||
cat "$json_path"
|
||||
'';
|
||||
}
|
||||
|
|
|
@ -13,9 +13,13 @@ logging.getLogger().setLevel(logging.DEBUG)
|
|||
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
|
||||
logging.getLogger(__name__).debug("logging enabled")
|
||||
|
||||
url = coerce_url(url, default_scheme="https")
|
||||
items = search(url, total_timeout=180, request_timeout=90, max_content_length=100*1024*1024)
|
||||
items = sort_urls(items)
|
||||
def try_scheme(url: str, scheme: str):
|
||||
url = coerce_url(url, default_scheme=scheme)
|
||||
print(f"trying {url}")
|
||||
items = search(url, total_timeout=180, request_timeout=90, max_content_length=100*1024*1024)
|
||||
return sort_urls(items)
|
||||
|
||||
items = try_scheme(url, "https") or try_scheme(url, "http")
|
||||
|
||||
# print all results
|
||||
serialized = [item.serialize() for item in items]
|
||||
|
|
Loading…
Reference in New Issue
Block a user