Commit 28bb4b287d50cae20be08de18dfde4ff128afabc
1 parent
0511eed7
Exists in
master
and in
2 other branches
API for the cache cleanup.
Showing
2 changed files
with
48 additions
and
4 deletions
Show diff stats
CHANGELOG.md
@@ -2,6 +2,7 @@ | @@ -2,6 +2,7 @@ | ||
2 | 2 | ||
3 | - [ ] Support multiple models for each target | 3 | - [ ] Support multiple models for each target |
4 | - [ ] Play button to start time dimension (not very useful?) | 4 | - [ ] Play button to start time dimension (not very useful?) |
5 | +- [ ] Optimize CSV generation (with some vectorization using numpy) | ||
5 | 6 | ||
6 | ## 1.0.0 | 7 | ## 1.0.0 |
7 | 8 | ||
@@ -13,11 +14,12 @@ | @@ -13,11 +14,12 @@ | ||
13 | - [x] Start/Stop datetime fields | 14 | - [x] Start/Stop datetime fields |
14 | - [x] Retry CSV generation when it fails due to a bug in AMDA's API | 15 | - [x] Retry CSV generation when it fails due to a bug in AMDA's API |
15 | - [x] Remove duplicate NetCDFs from AMDA's API response | 16 | - [x] Remove duplicate NetCDFs from AMDA's API response |
16 | -- [ ] Optimize CSV generation (with some vectorization using numpy) | 17 | +- [ ] Add a version number somewhere |
18 | +- [ ] Move the visitors counter to the footer | ||
17 | - [ ] Cache cleanup | 19 | - [ ] Cache cleanup |
18 | - - [ ] API at /cache/cleanup | 20 | + - [x] API at /cache/clear |
19 | - [ ] CRON statement to call it | 21 | - [ ] CRON statement to call it |
20 | -- [ ] Download raw data (as CSV) for current time interval and targets | 22 | +- [ ] Download raw data (tarball of CSV) for current time interval and targets |
21 | - [ ] Same via SAMP | 23 | - [ ] Same via SAMP |
22 | - [ ] Credit the author of the pixel art planets | 24 | - [ ] Credit the author of the pixel art planets |
23 | - [ ] Set the log level to _error_ in production (see `web/run.py`) | 25 | - [ ] Set the log level to _error_ in production (see `web/run.py`) |
web/run.py
@@ -4,7 +4,7 @@ import StringIO | @@ -4,7 +4,7 @@ import StringIO | ||
4 | from math import sqrt | 4 | from math import sqrt |
5 | 5 | ||
6 | from os import listdir, environ, remove as removefile | 6 | from os import listdir, environ, remove as removefile |
7 | -from os.path import isfile, join, abspath, dirname | 7 | +from os.path import isfile, join, abspath, dirname, isdir |
8 | 8 | ||
9 | import csv | 9 | import csv |
10 | import json | 10 | import json |
@@ -368,6 +368,34 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at): | @@ -368,6 +368,34 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at): | ||
368 | abort(500, "Failed creating CSV '%s' : %s" % (filename, e)) | 368 | abort(500, "Failed creating CSV '%s' : %s" % (filename, e)) |
369 | 369 | ||
370 | 370 | ||
371 | +def remove_files_created_before(date, in_directory): | ||
372 | + """ | ||
373 | + Will throw if something horrible happens, like invalid parameters. | ||
374 | + :param date: datetime object | ||
375 | + :param in_directory: | ||
376 | + :return: | ||
377 | + """ | ||
378 | + import os | ||
379 | + import time | ||
380 | + | ||
381 | + secs = time.mktime(date.timetuple()) | ||
382 | + | ||
383 | + if not isdir(in_directory): | ||
384 | + raise ValueError("Directory to clean '%s' does not exist.") | ||
385 | + | ||
386 | + removed_files = [] | ||
387 | + for file_name in os.listdir(in_directory): | ||
388 | + file_path = os.path.join(in_directory, file_name) | ||
389 | + if not os.path.isfile(file_path): | ||
390 | + continue | ||
391 | + t = os.stat(file_path) | ||
392 | + if t.st_ctime < secs: | ||
393 | + removed_files.append(file_path) | ||
394 | + os.remove(file_path) | ||
395 | + | ||
396 | + return removed_files | ||
397 | + | ||
398 | + | ||
371 | def increment_hit_counter(): | 399 | def increment_hit_counter(): |
372 | hit_count_path = get_path("../VISITS") | 400 | hit_count_path = get_path("../VISITS") |
373 | 401 | ||
@@ -494,6 +522,20 @@ def download_targets_tarball(targets, started_at, stopped_at): | @@ -494,6 +522,20 @@ def download_targets_tarball(targets, started_at, stopped_at): | ||
494 | 522 | ||
495 | return send_from_directory(get_path("../cache/"), gzip_filename) | 523 | return send_from_directory(get_path("../cache/"), gzip_filename) |
496 | 524 | ||
525 | + | ||
526 | +# API ######################################################################### | ||
527 | + | ||
528 | +@app.route("/cache/clear") | ||
529 | +def cache_clear(): | ||
530 | + """ | ||
531 | + Removes all files from the cache that are older than roughly one month. | ||
532 | + """ | ||
533 | + a_month_ago = datetime.datetime.now() - datetime.timedelta(days=32) | ||
534 | + cache_dir = get_path('../cache') | ||
535 | + removed_files = remove_files_created_before(a_month_ago, cache_dir) | ||
536 | + return "Cache cleared! Removed %d old file(s)." % len(removed_files) | ||
537 | + | ||
538 | + | ||
497 | # DEV TOOLS ################################################################### | 539 | # DEV TOOLS ################################################################### |
498 | 540 | ||
499 | # @app.route("/inspect") | 541 | # @app.route("/inspect") |