matt/spl-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'auto_generic'

Browse Source
This commit is contained in:
Kim Taylor
2024-09-24 00:00:00 +10:00
parent 03652e1285 6df6cc2d5e
commit a30855ac30
4 changed files with 304 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
csv-generic/gen-generic.php Normal file
View File

@@ -0,0 +1,101 @@
<?php
require_once("parse_generic_csv.php");
$options = getopt("", ["generic-csv:", "config-files:"]);
if (isset($options['generic-csv'])) {
$generic_csv = $options['generic-csv'];
} else {
error_log("Error: Missing required option '--generic-csv'.");
exit(1);
}
if (isset($options['config-files'])) {
$config_files = $options['config-files'];
} else {
error_log("Error: Missing required option '--config-files'.");
exit(1);
}
$config_files = explode(" ", $config_files);
$candidate_data = parse_generic_csv($generic_csv);
$lga_list = [];
/* Generate dictionary of LGAs and Wards */
foreach ($config_files as $config_file) {
$config_string = file_get_contents($config_file);
if ($config_string !== FALSE) {
$config = json_decode($config_string, true);
} else {
error_log("Error opening config.json.");
exit(1);
}
$config['config-file'] = $config_file;
$lga_list[] = $config;
}
/* Match user typed LGA/Ward to our database */
match_lga($candidate_data, $lga_list);
/* Calculate score for candidate */
foreach ($candidate_data as &$candidate) {
$score = 0;
if ($candidate['Pledge'] === "Yes") $score++;
if ($candidate['q1'] === "Yes") $score++;
if ($candidate['q3'] === "Yes") $score++;
if ($candidate['q4'] === "Yes") $score++;
if ($candidate['q7'] === "Yes") $score++;
$candidate['Score'] = $score;
}
$header = ["Ward", "Candidate Name", "Rating", "Pledge", "Picture"];
/* Generate candidates-generic.csv */
foreach ($lga_list as $lga) {
$lga_candidates = array_filter($candidate_data, function ($candidate) use ($lga) {
return $candidate['match_lga'] === $lga['slug'];
});
if (count($lga_candidates) === 0) continue;
$dir = dirname($lga['config-file']);
$dir_files = scandir($dir);
$output_file = $dir."/candidates-generic.csv";
if (($handle = fopen($output_file, "w")) === FALSE) {
error_log('Error opening output file');
exit(1);
}
if (fputcsv($handle, $header) === FALSE) {
error_log('Error writing headers to output file');
exit(3);
}
foreach ($lga_candidates as $candidate) {
/* Add extension to photo hash */
foreach ($dir_files as $file) {
if (strstr($file, $candidate['Photo'])) {
$candidate['Photo'] = $file;
}
}
$fields = [
$candidate['match_ward'],
$candidate['Name'],
$candidate['Score'],
$candidate['Pledge'],
$candidate['Photo'],
];
if (fputcsv($handle, $fields) === FALSE) {
error_log('Error writing candidate to output file');
exit(3);
}
}
}
exit(0);

54
csv-generic/gen-image-map.php Normal file
View File

@@ -0,0 +1,54 @@
<?php
require_once("parse_generic_csv.php");
$options = getopt("", ["generic-csv:", "config-files:"]);
if (isset($options['generic-csv'])) {
$generic_csv = $options['generic-csv'];
} else {
error_log("Error: Missing required option '--generic-csv'.");
exit(1);
}
if (isset($options['config-files'])) {
$config_files = $options['config-files'];
} else {
error_log("Error: Missing required option '--config-files'.");
exit(1);
}
$config_files = explode(" ", $config_files);
$candidate_data = parse_generic_csv($generic_csv);
$lga_list = [];
/* Generate dictionary of LGAs and Wards */
foreach ($config_files as $config_file) {
$config_string = file_get_contents($config_file);
if ($config_string !== FALSE) {
$config = json_decode($config_string, true);
} else {
error_log("Error opening config.json.");
exit(1);
}
$config['config-file'] = $config_file;
$lga_list[] = $config;
}
/* Match user typed LGA/Ward to our database */
match_lga($candidate_data, $lga_list);
$image_map = [];
foreach ($candidate_data as $candidate) {
if (strlen($candidate['photo_url'])) {
$map['url'] = $candidate['photo_url'];
$map['match_lga'] = $candidate['match_lga'];
$image_map[$candidate['Photo']] = $map;
}
}
$json_data = json_encode($image_map);
print_r($json_data);
exit(0);

88
csv-generic/parse_generic_csv.php Normal file
View File

@@ -0,0 +1,88 @@
<?php
function parse_generic_csv($generic_csv) {
$candidate_data = [];
if (($handle = fopen($generic_csv, "r")) !== FALSE) {
$headers = fgetcsv($handle);
while (($data = fgetcsv($handle)) !== FALSE) {
$candidate = [];
$question_no = 0;
$is_question = false;
foreach ($headers as $key => $value) {
/* Override key name for questions */
if ($value === "Verified") {
$is_question = false;
}
if (strstr($value, "candidate photo")) $value = "Photo";
if (strstr($value, "In which Local Government Area")) $value = "LGA";
if (strstr($value, "In which Ward")) $value = "Ward";
if (strstr($value, "Political Party")) $value = "Party";
if ($value === "Pledge") {
if (strstr($data[$key], "I pledge")) $data[$key] = "y";
else $data[$key] = "n";
}
if ($value === "Photo") {
$candidate['photo_url'] = $data[$key];
$data[$key] = preg_filter("/.*id=/", "", $data[$key]);
}
if ($is_question) {
$candidate['q'.$question_no++] = $data[$key];
} else {
$candidate[$value] = $data[$key];
}
if ($value === "Pledge") {
$is_question = true;
}
}
$candidate_data[] = $candidate;
}
fclose($handle);
} else {
error_log('Error opening candidates file');
exit(1);
}
return $candidate_data;
}
function match_lga(&$candidate_data, $lga_list) {
foreach ($candidate_data as &$candidate) {
/* Match user typed LGA/Ward to our database */
$max_score = 0;
foreach ($lga_list as $lga) {
$aa = preg_split("/[^a-z]/", strtolower($candidate['LGA']));
$bb = preg_split("/[^a-z]/", $lga['slug']);
$score_sum = 0;
foreach ($aa as $a) {
foreach ($bb as $b) {
similar_text($a, $b, $score);
if ($score > 70) $score_sum += $score;
else $score_sum -= 10;
}
}
if ($score_sum > $max_score) {
$max_score = $score_sum;
$match_lga = $lga;
}
}
$max_score = 0;
foreach ($match_lga['wardNames'] as $ward) {
similar_text($ward, $candidate['Ward'], $score);
if ($score > $max_score) {
$max_score = $score;
$match_ward = $ward;
}
}
$candidate['match_lga'] = $match_lga['slug'];
$candidate['match_ward'] = $match_ward;
}
}

61
get-generic.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
#rclone sync --progress bikewest:spl_generic_survey_2024 $DATA_LOC/google-data
GENERIC_SURVEY=../generic-survey/responses.csv
IMAGES=../generic-survey/images
DATA_PATH="../spl-data"
echo "Fetching latest responses to generic survey."
rclone -v copyto --drive-export-formats csv 'bikewest:spl_generic_survey_2024/Streets People Love council election candidate pledge and survey (Responses).csv' $GENERIC_SURVEY
config_files=()
for folder in "$DATA_PATH"/*; do
if test -f "$folder"/config.json; then
config_files+=("$folder"/config.json)
fi
done
image_map=$(php csv-generic/gen-image-map.php --generic-csv $GENERIC_SURVEY --config-files "${config_files[*]}")
img_list=()
for key in $(jq -r 'keys[]' <<< $image_map) ; do
if [ -f $IMAGES/$key ] ; then
continue
fi
img_list+=($key)
img_list+=($IMAGES/$key)
done
if [ ${#img_list[*]} -gt 0 ] ; then
echo "Downloading $((${#img_list[*]}/2)) image(s)..."
rclone -v backend copyid bikewest: ${img_list[*]}
fi
for key in $(jq -r 'keys[]' <<< $image_map) ; do
format=$(identify $IMAGES/$key | awk '{print $2}')
case $format in
PNG ) suffix=.png ;;
JPEG ) suffix=.jpg ;;
HEIC ) suffix=.jpg ;;
WEBP ) suffix=.png ;;
*)
echo "Error: Unknown image format: $IMAGES/$key"
;;
esac
lga=$(jq -r ".[\"$key\"][\"match_lga\"]" <<< $image_map)
dst="$DATA_PATH/$lga/$key$suffix"
if [ -f $dst ] ; then
continue
fi
echo "Resizing $dst"
convert $IMAGES/$key -resize 400x400 $dst
done
echo "Generating candidates-generic.csv files."
php csv-generic/gen-generic.php --generic-csv $GENERIC_SURVEY --config-files "${config_files[*]}"