From 3adc553a0729ed5f01ce3ee586031ee6244d6c74 Mon Sep 17 00:00:00 2001
From: Thomas Lambert <t.lambert@uliege.be>
Date: Wed, 5 Jul 2023 17:02:24 +0200
Subject: [PATCH] feat(scripts): add UIUC propeller DB downloader
---
.gitignore | 1 +
miss_hit.cfg | 33 +++++++
scripts/uiucdownload.m | 192 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 226 insertions(+)
create mode 100644 .gitignore
create mode 100644 miss_hit.cfg
create mode 100644 scripts/uiucdownload.m
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ca85a04
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+database/UIUC-propDB*
diff --git a/miss_hit.cfg b/miss_hit.cfg
new file mode 100644
index 0000000..d53390c
--- /dev/null
+++ b/miss_hit.cfg
@@ -0,0 +1,33 @@
+# miss_hit style checker
+# (https://florianschanda.github.io/miss_hit/style_checker.html)
+#
+project_root
+
+# Ensure copyright notice is everywhere
+copyright_primary_entity: "University of Liege"
+copyright_entity: "Thomas Lambert <t.lambert@uliege.be>"
+
+# Enable miss_hit on the whole repo
+enable: 1
+enforce_encoding: "utf8"
+
+# Layout
+line_length: 100
+tab_width: 4
+
+# Regex to enforce naming convention
+regex_script_name: "[a-z](_?[a-z0-9]+)*"
+regex_function_name: "[a-z](_?[a-z0-9]+)*"
+regex_nested_name: "[a-z](_?[a-z0-9]+)*"
+regex_parameter_name: "[a-zA-Z](_?[a-zA-Z0-9]+)*"
+regex_class_name: "[A-Z]([a-zA-Z0-9]+)*"
+regex_method_name: "[a-z](_?[a-zA-Z0-9]+)*"
+
+# metrics limit for the code quality
+# (https://florianschanda.github.io/miss_hit/metrics.html)
+metric "cnest": limit 5
+metric "file_length": limit 800
+metric "cyc": limit 15
+metric "parameters": limit 10
+metric "globals": limit 0
+metric "persistent": limit 3
diff --git a/scripts/uiucdownload.m b/scripts/uiucdownload.m
new file mode 100644
index 0000000..9ce2879
--- /dev/null
+++ b/scripts/uiucdownload.m
@@ -0,0 +1,192 @@
+function uiucdownload(varargin)
+ % UIUCDOWNLOAD Download, extract and clean the UIUC propeller database.
+ % This function executes the following tasks:
+ % 1. Download the UIUC Propeller database into database/UIUC-propDB_YYYYMMDD.zip. It also
+ % saves the file hash into a textfile in oder to prevent useless re-downloads.
+ % 2. Unzip the database and moves the data files found in volumeX/data/* into
+ % database/UIUC_propDB/volumeX/*
+ % 3. The rest of the unzipped files are cleaned up
+ % 4. The zip file is either kept or removed, based on user input
+ % -----
+ %
+ % Syntax:
+ % uiucdownload() Download the database (if needed) and its md5 checksum, extract the data
+ % files and cleanup afterwards.
+ %
+ % uiucdownload('force', true) Forces the redownload of the database, even if the md5 checksum
+ % found locally matches the one present on the UIUC website. Default: false.
+ %
+ % uiucdownload('quiet', true) Removes all verbosity from the command line. Default: false.
+ %
+ % uiucdownload('removeZip', true) Removes the zipfile as well during cleanup. Default: true.
+ %
+ % uiucdownload('removeSum', true) Removes the checksum file as well during cleanup.
+ % Note that this will result in a forced download of the database if the function is run
+ % again, even if the zip-file is still present locally. Default: false.
+ %
+ % Inputs:
+ % 'force' : Forces redownload of the database
+ % 'quiet' : Removes verbosity from the command line
+ % 'removeZip' : Removes the zip-file at cleanup
+ % 'removeSum' : Removes the checksum at cleanup. This will force a redownload if the function
+ % is run again
+ %
+ % Outputs:
+ % UIUC-propDB : Directory containing the all useful data from the UIUC propeller database.
+ %
+ % Examples:
+ % uiucdownload()
+ % uiucdownload('force', true)
+ % uiucdownload('removeZip', false, 'force', true)
+
+ % -----------------------------------------
+ % (c) Copyright 2023 University of Liege
+ % Author: Thomas Lambert <t.lambert@uliege.be>
+ % ULiege - Aeroelasticity and Experimental Aerodynamics
+ % MIT License
+ % Repo: https://gitlab.uliege.be/rotare/rotor-db
+ % Issues: https://gitlab.uliege.be/rotare/rotor-db/-/issues
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+ close all;
+ clc;
+
+ % Defaults and constants
+ PROP_DB_URL = 'https://m-selig.ae.illinois.edu/props/propDB.html';
+ PROP_DB_ZIP_URL = 'https://m-selig.ae.illinois.edu/props/download/UIUC-propDB.zip';
+ N_VOLUMES = 4; % Number of volumes in the database
+
+ DB_DIR = '../database/';
+ DB_NAME = 'UIUC-propDB';
+ ZIP_FILENAME = 'UIUC-propDB.zip';
+ SUM_FILENAME = 'UIUC-propDB.md5';
+
+ % Parse function options
+ [forceDownload, removeZip, removeSum, quiet] = parseoptions(varargin{:});
+
+ % Retreive the md5sum of the UIUC-propDB.zip
+ sumfilename = [DB_DIR, SUM_FILENAME];
+ currentSum = getcurrentchecksum(PROP_DB_URL); % Checksum from the website
+ oldSum = getlocalchecksum(sumfilename); % Checksum saved locally
+
+ if ~strcmp(currentSum, oldSum) || forceDownload
+ zipfilename = [DB_DIR, ZIP_FILENAME];
+ tempdir = [zipfilename, '.temp'];
+ dbname = [DB_DIR, DB_NAME];
+
+ % Download database and save corresponding checksum
+ verboseoutput(quiet, 'Downloading the database. This may take a while...\n');
+ websave([DB_DIR, ZIP_FILENAME], PROP_DB_ZIP_URL); % TODO UNCOMMENT AFTER TESTING
+ verboseoutput(quiet, 'Download completed.\n');
+ savetofile(sumfilename, currentSum);
+
+ % Unzip database and copy files
+ verboseoutput(quiet, 'Extracting database files... ');
+ unzip(zipfilename, tempdir);
+ verboseoutput(quiet, 'Done!\nCopying the database files...');
+ copydbfiles(tempdir, dbname, N_VOLUMES);
+ verboseoutput(quiet, 'Done!\n');
+
+ % Cleanup
+ verboseoutput(quiet, 'Cleaning up...\n');
+ cleanup(tempdir, zipfilename, sumfilename, removeZip, removeSum);
+ verboseoutput(quiet, 'All done!\n');
+ else
+ verboseoutput(quiet, ['It seems that the database has already been downloaded.\n'...
+ 'You can force a re-download by using '...
+ '''uiucdownload(''force'', true)''.\n']);
+ end
+
+end
+
+% --------------------------------------------------------------------------------------------------
+function [forceDownload, removeZip, removeSum, quiet] = parseoptions(varargin)
+ % PARSEOPTIONS Parse function optional inputs
+
+ % Defaults and constants
+ DEF_FORCE_DOWNLOAD = false; % Force redownload of the full DB
+ DEF_REMOVE_ZIP = true; % Remove the zipfile during cleanup
+ DEF_REMOVE_SUM = false; % Remove the checksum during cleanup (not advised)
+ DEF_QUIET = false; % Do not show any output
+
+ % Option validators
+ validLogical = @(x) validateattributes(x, {'logical'}, {'scalar'});
+
+ % Parse options
+ p = inputParser;
+ addParameter(p, 'force', DEF_FORCE_DOWNLOAD, validLogical);
+ addParameter(p, 'quiet', DEF_QUIET, validLogical);
+ addParameter(p, 'removeZip', DEF_REMOVE_ZIP, validLogical);
+ addParameter(p, 'removeSum', DEF_REMOVE_SUM, validLogical);
+ parse(p, varargin{:});
+ forceDownload = p.Results.force;
+ removeZip = p.Results.removeZip;
+ removeSum = p.Results.removeSum;
+ quiet = p.Results.quiet;
+end
+
+function checksum = getcurrentchecksum(url)
+ % GETCURRENTCHECKSUM Returns the file checksum found on the webpage
+
+ MD5_REGEX = '([a-fA-F\d]{32})';
+
+ % Read and parse webpage
+ webpage = webread(url);
+ parsedPage = textscan(webpage, '%s', 'delimiter', '\n');
+ % Get current checksum
+ checksumLine = parsedPage{1}(contains(parsedPage{1}(:), "md5sum"));
+ [md5start, md5end] = regexp(checksumLine{:}, MD5_REGEX);
+ checksum = checksumLine{:}(md5start:md5end);
+end
+
+function checksum = getlocalchecksum(file)
+ % GETLOCALCHECKSUM Retreive checksum saved locally
+
+ checksum = [];
+ if isfile(file)
+ checksum = fileread(file);
+ end
+end
+
+function savetofile(file, string)
+ % SAVETOFILE Save a string to a file
+
+ fileID = fopen(file, 'w');
+ fprintf(fileID, '%s', string);
+ fclose(fileID);
+end
+
+function copydbfiles(tempdir, dbname, nVolumes)
+ % COPYDBFILES Copy the (interesting) database files
+
+ for iVol = 1:nVolumes
+ volumeDir = [dbname, '/volume-', num2str(iVol)];
+ mkdir(volumeDir);
+ copyfile([tempdir, '/UIUC-propDB/volume-', num2str(iVol), '/data/*'], volumeDir);
+ end
+end
+
+function verboseoutput(quiet, string)
+ % VERBOSEOUTPUT Output information to the console
+
+ if ~quiet
+ fprintf(string);
+ end
+end
+
+function cleanup(tempdir, zipfilename, sumfilename, removeZip, removeSum)
+ % CLEANUP Clean-up directories, files, etc
+
+ % Remove the extracted archive
+ rmdir(tempdir, 's');
+
+ % Remove zip archive
+ if removeZip
+ delete(zipfilename);
+ end
+
+ % Remove checksum
+ if removeSum
+ delete(sumfilename);
+ end
+end
--
GitLab