Import some CSVs

This commit is contained in:
Brandon Dyck 2023-04-18 20:36:51 -06:00
commit ff7b992aac
5 changed files with 93 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
.venv
*.db

45
createdb.sh Normal file
View File

@ -0,0 +1,45 @@
#!/bin/sh
set -e
DB=francecrime.db
DEPT=department
REG=region
DEPT_CRIMES=department_crimes
rm -f "$DB"
sqlite-utils create-database "$DB"
# Region lookup table
curl -L https://www.insee.fr/fr/statistiques/fichier/6800675/v_region_2023.csv | \
sqlite-utils insert "$DB" "$REG" - --csv --convert 'return {"code": row["REG"], "name": row["NCCENR"]}'
sqlite-utils transform "$DB" "$REG" --pk code
# Department lookup table
curl -L https://www.insee.fr/fr/statistiques/fichier/6800675/v_departement_2023.csv | \
sqlite-utils insert "$DB" "$DEPT" - --csv --convert 'return {"code": row["DEP"], "name": row["NCCENR"]}'
sqlite-utils transform "$DB" "$DEPT" --pk code
# TODO Add a way to filter out island possessions
# Crimes by department
curl -L https://www.data.gouv.fr/fr/datasets/r/acc332f6-92be-42af-9721-f3609bea8cfc | gzip -d | \
sqlite-utils insert "$DB" "$DEPT_CRIMES" - --csv --delimiter=";" --convert '
def full_year(year):
return "20" + year
def comma_float(s):
return float(s.replace(",", "."))
return {
"class": row["classe"],
"year": full_year(row["annee"]),
"department": row["Code.département"],
"region": row["Code.région"],
"cases": int(row["faits"]),
"population": int(row["POP"]),
"rate_per_thousand": comma_float(row["tauxpourmille"])}'
sqlite-utils extract "$DB" "$DEPT_CRIMES" class
sqlite-utils transform "$DB" "$DEPT_CRIMES" --pk class_id --pk year --pk department
sqlite-utils add-foreign-keys "$DB" \
"$DEPT_CRIMES" department "$DEPT" code \
"$DEPT_CRIMES" region "$REG" code
# TODO Rename class_id to class and show English text using crime_classes.csv
datasette "$DB" -o

13
crime_classes.csv Normal file
View File

@ -0,0 +1,13 @@
class,class_french
Intentional blows and injuries,Coups et blessures volontaires
Intrafamilial intentional blows and injuries,Coups et blessures volontaires intrafamiliaux
Other intentional blows and injuries,Autres coups et blessures volontaires
Sexual violence,Violences sexuelles
Armed robberies,Vols avec armes
Violent robberies without a weapon,Vols violents sans arme
Nonviolent thefts,Vols sans violence contre des personnes
Home burglaries,Cambriolages de logement
Thefts of vehicles,Vols de véhicules
Thefts from vehicles,Vols dans les véhicules
Thefts of vehicle accessories,Vols d'accessoires sur véhicules
Intentional destruction and damage,Destructions et dégradations volontaires
1 class class_french
2 Intentional blows and injuries Coups et blessures volontaires
3 Intrafamilial intentional blows and injuries Coups et blessures volontaires intrafamiliaux
4 Other intentional blows and injuries Autres coups et blessures volontaires
5 Sexual violence Violences sexuelles
6 Armed robberies Vols avec armes
7 Violent robberies without a weapon Vols violents sans arme
8 Nonviolent thefts Vols sans violence contre des personnes
9 Home burglaries Cambriolages de logement
10 Thefts of vehicles Vols de véhicules
11 Thefts from vehicles Vols dans les véhicules
12 Thefts of vehicle accessories Vols d'accessoires sur véhicules
13 Intentional destruction and damage Destructions et dégradations volontaires

BIN
department.csv Normal file

Binary file not shown.
Can't render this file because it has a wrong number of fields in line 2.

33
requirements.txt Normal file
View File

@ -0,0 +1,33 @@
aiofiles==23.1.0
anyio==3.6.2
asgi-csrf==0.9
asgiref==3.6.0
certifi==2022.12.7
click==8.1.3
click-default-group-wheel==1.2.2
datasette==0.64.1
h11==0.14.0
httpcore==0.16.3
httpx==0.23.3
hupper==1.11
idna==3.4
itsdangerous==2.1.2
janus==1.0.0
Jinja2==3.1.2
lxml==4.9.2
MarkupSafe==2.1.2
mergedeep==1.3.4
Pint==0.20.1
pluggy==1.0.0
python-dateutil==2.8.2
python-multipart==0.0.5
PyYAML==6.0
rfc3986==1.5.0
six==1.16.0
sniffio==1.3.0
sqlglot==11.0.1
sqlite-fts4==1.0.3
sqlite-utils==3.30
tabulate==0.9.0
typing_extensions==4.4.0
uvicorn==0.20.0