Commit 28bb4b287d50cae20be08de18dfde4ff128afabc

Authored by Goutte
1 parent 0511eed7

API for the cache cleanup.

Showing 2 changed files with 48 additions and 4 deletions   Show diff stats
@@ -2,6 +2,7 @@ @@ -2,6 +2,7 @@
2 2
3 - [ ] Support multiple models for each target 3 - [ ] Support multiple models for each target
4 - [ ] Play button to start time dimension (not very useful?) 4 - [ ] Play button to start time dimension (not very useful?)
  5 +- [ ] Optimize CSV generation (with some vectorization using numpy)
5 6
6 ## 1.0.0 7 ## 1.0.0
7 8
@@ -13,11 +14,12 @@ @@ -13,11 +14,12 @@
13 - [x] Start/Stop datetime fields 14 - [x] Start/Stop datetime fields
14 - [x] Retry CSV generation when it fails due to a bug in AMDA's API 15 - [x] Retry CSV generation when it fails due to a bug in AMDA's API
15 - [x] Remove duplicate NetCDFs from AMDA's API response 16 - [x] Remove duplicate NetCDFs from AMDA's API response
16 -- [ ] Optimize CSV generation (with some vectorization using numpy) 17 +- [ ] Add a version number somewhere
  18 +- [ ] Move the visitors counter to the footer
17 - [ ] Cache cleanup 19 - [ ] Cache cleanup
18 - - [ ] API at /cache/cleanup 20 + - [x] API at /cache/clear
19 - [ ] CRON statement to call it 21 - [ ] CRON statement to call it
20 -- [ ] Download raw data (as CSV) for current time interval and targets 22 +- [ ] Download raw data (tarball of CSV) for current time interval and targets
21 - [ ] Same via SAMP 23 - [ ] Same via SAMP
22 - [ ] Credit the author of the pixel art planets 24 - [ ] Credit the author of the pixel art planets
23 - [ ] Set the log level to _error_ in production (see `web/run.py`) 25 - [ ] Set the log level to _error_ in production (see `web/run.py`)
@@ -4,7 +4,7 @@ import StringIO @@ -4,7 +4,7 @@ import StringIO
4 from math import sqrt 4 from math import sqrt
5 5
6 from os import listdir, environ, remove as removefile 6 from os import listdir, environ, remove as removefile
7 -from os.path import isfile, join, abspath, dirname 7 +from os.path import isfile, join, abspath, dirname, isdir
8 8
9 import csv 9 import csv
10 import json 10 import json
@@ -368,6 +368,34 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at): @@ -368,6 +368,34 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at):
368 abort(500, "Failed creating CSV '%s' : %s" % (filename, e)) 368 abort(500, "Failed creating CSV '%s' : %s" % (filename, e))
369 369
370 370
  371 +def remove_files_created_before(date, in_directory):
  372 + """
  373 + Will throw if something horrible happens, like invalid parameters.
  374 + :param date: datetime object
  375 + :param in_directory:
  376 + :return:
  377 + """
  378 + import os
  379 + import time
  380 +
  381 + secs = time.mktime(date.timetuple())
  382 +
  383 + if not isdir(in_directory):
  384 + raise ValueError("Directory to clean '%s' does not exist.")
  385 +
  386 + removed_files = []
  387 + for file_name in os.listdir(in_directory):
  388 + file_path = os.path.join(in_directory, file_name)
  389 + if not os.path.isfile(file_path):
  390 + continue
  391 + t = os.stat(file_path)
  392 + if t.st_ctime < secs:
  393 + removed_files.append(file_path)
  394 + os.remove(file_path)
  395 +
  396 + return removed_files
  397 +
  398 +
371 def increment_hit_counter(): 399 def increment_hit_counter():
372 hit_count_path = get_path("../VISITS") 400 hit_count_path = get_path("../VISITS")
373 401
@@ -494,6 +522,20 @@ def download_targets_tarball(targets, started_at, stopped_at): @@ -494,6 +522,20 @@ def download_targets_tarball(targets, started_at, stopped_at):
494 522
495 return send_from_directory(get_path("../cache/"), gzip_filename) 523 return send_from_directory(get_path("../cache/"), gzip_filename)
496 524
  525 +
  526 +# API #########################################################################
  527 +
  528 +@app.route("/cache/clear")
  529 +def cache_clear():
  530 + """
  531 + Removes all files from the cache that are older than roughly one month.
  532 + """
  533 + a_month_ago = datetime.datetime.now() - datetime.timedelta(days=32)
  534 + cache_dir = get_path('../cache')
  535 + removed_files = remove_files_created_before(a_month_ago, cache_dir)
  536 + return "Cache cleared! Removed %d old file(s)." % len(removed_files)
  537 +
  538 +
497 # DEV TOOLS ################################################################### 539 # DEV TOOLS ###################################################################
498 540
499 # @app.route("/inspect") 541 # @app.route("/inspect")