From c27cc2831b3416e53554e4221b65435c33159ba6 Mon Sep 17 00:00:00 2001
From: Kim Taylor <kmtaylor@gmx.com>
Date: Mon, 23 Sep 2024 23:25:12 +1000
Subject: [PATCH] Image fetch and resize working.

---
 csv-generic/gen-generic.php       | 12 +++++++-
 csv-generic/parse_generic_csv.php |  4 +--
 get-generic.sh                    | 50 +++++++++++++++++++++++++------
 3 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/csv-generic/gen-generic.php b/csv-generic/gen-generic.php
index a0901ce..337b991 100644
--- a/csv-generic/gen-generic.php
+++ b/csv-generic/gen-generic.php
@@ -61,7 +61,9 @@ foreach ($lga_list as $lga) {
 
     if (count($lga_candidates) === 0) continue;
 
-    $output_file = dirname($lga['config-file'])."/candidates-generic.csv";
+    $dir = dirname($lga['config-file']);
+    $dir_files = scandir($dir);
+    $output_file = $dir."/candidates-generic.csv";
 
     if (($handle = fopen($output_file, "w")) === FALSE) {
         error_log('Error opening output file');
@@ -74,10 +76,18 @@ foreach ($lga_list as $lga) {
     }
 
     foreach ($lga_candidates as $candidate) {
+        /* Add extension to photo hash */
+        foreach ($dir_files as $file) {
+            if (strstr($file, $candidate['Photo'])) {
+                $candidate['Photo'] = $file;
+            }
+        }
+
         $fields = [
             $candidate['match_ward'],
             $candidate['Name'],
             $candidate['Score'],
+            $candidate['Pledge'],
             $candidate['Photo'],
         ];
 
diff --git a/csv-generic/parse_generic_csv.php b/csv-generic/parse_generic_csv.php
index c90ee05..9458c2d 100644
--- a/csv-generic/parse_generic_csv.php
+++ b/csv-generic/parse_generic_csv.php
@@ -20,8 +20,8 @@ function parse_generic_csv($generic_csv) {
                 if (strstr($value, "Political Party"))                  $value = "Party";
 
                 if ($value === "Pledge") {
-                    if (strstr($data[$key], "I pledge")) $data[$key] = "Yes";
-                    else $data[$key] = "No";
+                    if (strstr($data[$key], "I pledge")) $data[$key] = "y";
+                    else $data[$key] = "n";
                 }
                 
                 if ($value === "Photo") {
diff --git a/get-generic.sh b/get-generic.sh
index 28ef8ed..9a8686d 100755
--- a/get-generic.sh
+++ b/get-generic.sh
@@ -2,12 +2,14 @@
 
 #rclone sync --progress bikewest:spl_generic_survey_2024 $DATA_LOC/google-data
 
-#rclone --drive-export-formats csv copyto 'bikewest:spl_generic_survey_2024/Streets People Love council election candidate pledge and survey  (Responses).csv' $DATA_LOC/responses.csv
-
 GENERIC_SURVEY=../generic-survey/responses.csv
+IMAGES=../generic-survey/images
 
 DATA_PATH="../spl-data"
 
+echo "Fetching latest responses to generic survey."
+rclone -v copyto --drive-export-formats csv 'bikewest:spl_generic_survey_2024/Streets People Love council election candidate pledge and survey  (Responses).csv' $GENERIC_SURVEY
+
 config_files=()
 for folder in "$DATA_PATH"/*; do
     if test -f "$folder"/config.json; then
@@ -15,15 +17,45 @@ for folder in "$DATA_PATH"/*; do
     fi
 done
 
-
 image_map=$(php csv-generic/gen-image-map.php --generic-csv $GENERIC_SURVEY --config-files "${config_files[*]}")
 
+img_list=()
 for key in $(jq -r 'keys[]' <<< $image_map) ; do
-    url=$(jq -r ".[\"$key\"][\"url\"]" <<< $image_map)
-    lga=$(jq -r ".[\"$key\"][\"match_lga\"]" <<< $image_map)
-    dst=$DATA_PATH/$lga/$key
-    echo wget $url -O $dst
-    break
+    if [ -f $IMAGES/$key ] ; then
+        continue
+    fi
+    img_list+=($key)
+    img_list+=($IMAGES/$key)
 done
 
-#php csv-generic/gen-generic.php --generic-csv $GENERIC_SURVEY --config-files "${config_files[*]}"
+if [ ${#img_list[*]} -gt 0 ] ; then
+    echo "Downloading $((${#img_list[*]}/2)) image(s)..."
+    rclone -v backend copyid bikewest: ${img_list[*]}
+fi
+
+for key in $(jq -r 'keys[]' <<< $image_map) ; do
+    format=$(identify $IMAGES/$key | awk '{print $2}')
+
+    case $format in
+        PNG )  suffix=.png ;;
+        JPEG ) suffix=.jpg ;;
+        HEIC ) suffix=.jpg ;;
+        WEBP ) suffix=.png ;;
+        *)
+            echo "Error: Unknown image format: $IMAGES/$key"
+            ;;
+    esac
+
+    lga=$(jq -r ".[\"$key\"][\"match_lga\"]" <<< $image_map)
+    dst="$DATA_PATH/$lga/$key$suffix"
+
+    if [ -f $dst ] ; then
+        continue
+    fi
+
+    echo "Resizing $dst"
+    convert $IMAGES/$key -resize 400x400 $dst
+done
+
+echo "Generating candidates-generic.csv files."
+php csv-generic/gen-generic.php --generic-csv $GENERIC_SURVEY --config-files "${config_files[*]}"