Skip to content

Commit c901ba6

Browse files
ericocericoc
committed
upload utility usage files and import them live
1 parent 865b954 commit c901ba6

File tree

6 files changed

+314
-19
lines changed

6 files changed

+314
-19
lines changed

‎apps/core/forms.py

+193
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
from csv import DictReader
2+
from datetime import date, datetime, time
3+
from django.conf import settings
4+
from django.core.validators import FileExtensionValidator
5+
from django.forms import Form, FileField, ValidationError
6+
from django.utils import timezone
7+
from openpyxl import load_workbook
8+
from pathlib import Path
9+
from warnings import catch_warnings, filterwarnings
10+
11+
12+
class UploadUsageDataForm(Form):
13+
"""
14+
Form to handle upload of utility usage data file.
15+
"""
16+
file = FileField(
17+
validators=(
18+
FileExtensionValidator(
19+
allowed_extensions=settings.USAGE_FILE_SUFFIXES
20+
),
21+
)
22+
)
23+
24+
def clean_file(self):
25+
"""
26+
Validate various uploaded utility usage data files.
27+
"""
28+
file = self.cleaned_data.get("file")
29+
if not file:
30+
raise ValidationError("Missing file!")
31+
32+
encoding = "utf-8"
33+
file_path = Path(settings.MEDIA_ROOT, file.name)
34+
usage = []
35+
utility = None
36+
37+
_valid_units = {
38+
"electric": ("TYPE", "Electric usage", "kWh"),
39+
"water": (" Units", " Gallons"),
40+
}
41+
42+
# Get uploaded file name suffix, and validate MIME type.
43+
suffix = file_path.suffix.lower()
44+
45+
# Write uploaded file locally.
46+
with open(file_path, mode="wb+") as fh_w:
47+
for chunk in file.chunks():
48+
fh_w.write(chunk)
49+
50+
"""
51+
Handle the uploaded file (which was just written) uniquely per utility.
52+
"""
53+
54+
"""Electric or Water usage: comma-separated values (.csv)"""
55+
if suffix == ".csv":
56+
57+
# Different encoding for electric usage CSV file.
58+
if file.name == settings.WATER_FILENAME:
59+
utility = "water"
60+
61+
# Different encoding for electric usage CSV file.
62+
if file.name.startswith(settings.ELECTRIC_PREFIX):
63+
encoding += "-sig"
64+
utility = "electric"
65+
66+
# Open CSV file that we just wrote, with proper encoding.
67+
with open(file_path, mode="r", encoding=encoding) as read_fh:
68+
csv_lines = read_fh.readlines()
69+
to_read = csv_lines
70+
71+
if utility == "electric":
72+
73+
# Ensure electric usage (7th row, 5th column) is "kWh".
74+
uf = csv_lines[6].split(",")[4]
75+
unit_found = uf.split("(")[1].split(")")[0]
76+
assert unit_found == _valid_units[utility][2], (
77+
f"Invalid {utility} unit column!"
78+
)
79+
80+
# Skip header of electric usage file.
81+
to_read = csv_lines[6:]
82+
83+
# Get valid unit for electric or water usage CSV rows.
84+
utility_unit = _valid_units[utility][1]
85+
86+
# Read CSV of electric or water usage data, iterating rows.
87+
reader = DictReader(to_read)
88+
for row in reader:
89+
90+
# Electric or water CSV file rows must be valid units.
91+
row_unit = row[_valid_units[utility][0]]
92+
assert row_unit == utility_unit, (
93+
f"Invalid {utility} unit ({row_unit})!"
94+
)
95+
96+
"""
97+
Parse electric usage row.
98+
"""
99+
if utility == "electric":
100+
101+
# Parse date/time columns for each electric usage row.
102+
time_pcs = row["START TIME"].split(':')
103+
104+
# Map hour to electricity usage in floating point kWh.
105+
usage.append({
106+
"hour": timezone.make_aware(
107+
value=datetime.combine(
108+
date=date.fromisoformat(row["DATE"]),
109+
time=time(
110+
hour=int(time_pcs[0]),
111+
minute=int(time_pcs[1])
112+
)
113+
)
114+
),
115+
"kwh": float(row["USAGE (kWh)"])
116+
})
117+
118+
"""
119+
Parse water usage row.
120+
"""
121+
if utility == "water":
122+
123+
# Parse date column for each water usage row.
124+
date_pcs = row[" Time Interval"].strip().split('/')
125+
date_iso = f"{date_pcs[2]}-{date_pcs[0]}-{date_pcs[1]}"
126+
127+
# Map each day to water usage floating point gallons.
128+
usage.append({
129+
"day": date.fromisoformat(date_iso),
130+
"gallons": float(row[" Consumption"].strip())
131+
})
132+
133+
"""Natural Gas usage: Microsoft Excel (.xlsx)"""
134+
xlsx_prefix = settings.NATURAL_GAS_PREFIX
135+
if suffix == ".xlsx" and file.name.startswith(xlsx_prefix):
136+
utility = "natural_gas"
137+
138+
# Filter warnings about spreadsheet style.
139+
with catch_warnings():
140+
filterwarnings(
141+
action="ignore",
142+
category=UserWarning,
143+
module="openpyxl.styles.stylesheet"
144+
)
145+
146+
# Load/read the spreadsheet workbook.
147+
xlsx_wb = load_workbook(filename=file_path, read_only=True)
148+
book_obj = xlsx_wb.active
149+
sheet_obj = book_obj
150+
sheet_title = sheet_obj.title or ""
151+
152+
# Confirm worksheet title.
153+
assert sheet_title == xlsx_prefix, (
154+
f"Invalid worksheet ({sheet_title})!"
155+
)
156+
157+
# Gather all worksheet rows into a list.
158+
rows = list(sheet_obj.iter_rows())
159+
160+
# Ensure natural gas unit is "CCF" (5th row, 2nd column).
161+
unit_found = rows[4][1].value.split('(')[1].split(')')[0]
162+
assert unit_found == "CCF", f"Invalid {utility} unit!"
163+
164+
# Skip header rows to parse columns of natural gas usage data.
165+
# Bill Month, Units Consumed (CCF), Period Start, Period End
166+
for row in rows[5:]:
167+
"""
168+
Parse natural gas usage row.
169+
"""
170+
row_month, row_ccf, row_start, row_end = row
171+
172+
# Parse "Bill Month" column from each natural gas usage row.
173+
# Example: "Mar, 2025"
174+
row_dt = datetime.strptime(
175+
row_month.value, "%b, %Y"
176+
).date()
177+
178+
# Map each month to natural gas usage in CCF as floating point.
179+
usage.append({
180+
"month": row_dt,
181+
"ccf": float(row_ccf.value)
182+
})
183+
184+
if usage:
185+
self.cleaned_data["usage"] = usage
186+
187+
if utility:
188+
self.cleaned_data["utility"] = utility
189+
190+
# Delete the local copy of the uploaded usage data file.
191+
if file_path and file_path.is_file() and file_path.exists():
192+
if file_path.unlink():
193+
return file

‎apps/core/forms/__init__.py

-1
This file was deleted.

‎apps/core/forms/upload.py

-13
This file was deleted.

‎apps/core/templates/add.html

+33-4
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,41 @@
11
{% extends 'base.html' %}
22
{% block main %}
3-
<div class="border m-3 p-3 rounded">
4-
<form action="{% url 'add' %}" method="post">
5-
{{ form.as_p }}
3+
<div id="add">
4+
{% if form.errors %}
5+
<div class="alert alert-danger border list-group m-3 p-3 errors rounded">
6+
{% for error in form.errors.values %}
7+
<div class="list-group-item list-group-item-danger">{{ error }}</div>
8+
{% endfor %}
9+
</div>
10+
{% endif %}
11+
{% if messages %}
12+
<div class="alert border list-group m-3 p-3 messages rounded">
13+
{% for message in messages %}
14+
<div class="list-group-item list-group-item-{% if message.tags %}{{ message.tags }}{% else %}success{% endif %}">
15+
{{ message }}
16+
</div>
17+
{% endfor %}
18+
</div>
19+
{% endif %}
20+
{% if form %}
21+
<form action="{% url 'add' %}" class="border m-3 p-3 rounded" enctype="multipart/form-data" method="post">
22+
<h3 title="{{ title }}">
23+
{{ title }}
24+
</h3>
25+
<p class="form-text">
26+
Please upload a comma-separated values (<span class="font-monospace">csv</span> - electric or water)
27+
file, or Microsoft Excel (<span class="font-monospace">xlsx</span> - natural gas) spreadsheet
28+
to add new data.
29+
</p>
30+
<p>
31+
{{ form.file }}
32+
</p>
33+
{% csrf_token %}
634
<p>
7-
<input class="btn btn-outline-primary" type="submit" value="Upload!">
35+
<button class="btn btn-outline-primary" type="submit">{{ title }}</button>
836
</p>
937
</form>
38+
{% endif %}
1039
</div>
1140
{% endblock main %}
1241
{% block scripts %}{% endblock scripts %}

‎apps/core/views/add.py

+86-1
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,98 @@
11
from django.contrib.auth.mixins import LoginRequiredMixin
2+
from django.contrib import messages
3+
from django.forms import ValidationError
4+
from django.utils.html import format_html
5+
from django.utils.translation import ngettext
26
from django.views.generic.edit import FormView
37

4-
from ..forms.upload import UploadUsageDataForm
8+
from apps.electric.models import ElectricUsage
9+
from apps.natural_gas.models import NaturalGasUsage
10+
from apps.water.models import WaterUsage
11+
512
from .base import BaseView
13+
from ..forms import UploadUsageDataForm
14+
615

716
class AddView(LoginRequiredMixin, BaseView, FormView):
817
"""Add data via upload form view."""
918
color = "var(--bs-success)"
1019
form_class = UploadUsageDataForm
20+
http_method_names = ("get", "post")
21+
model = None
1122
success_url = "/add/"
1223
template_name = "add.html"
1324
title = "Add"
25+
26+
def form_valid(self, form):
27+
28+
model = None
29+
num_created = 0
30+
utility = form.cleaned_data.get("utility")
31+
if not utility:
32+
raise ValidationError("Missing utility!")
33+
34+
# Choose model uniquely per utility.
35+
if utility == "electric":
36+
model = ElectricUsage
37+
if utility == "natural_gas":
38+
model = NaturalGasUsage
39+
if utility == "water":
40+
model = WaterUsage
41+
42+
# Ensure that a model is chosen.
43+
if not model:
44+
raise ValidationError(f"Missing model (for {utility})!")
45+
46+
# Ensure that data was found from the uploaded utility usage file.
47+
usage = form.cleaned_data.get("usage")
48+
if not usage:
49+
raise ValidationError(f"Missing usage (for {utility}!")
50+
51+
# Count the number of usage records found in the uploaded file.
52+
num_found = len(usage)
53+
utility_title = utility.replace("_", " ")
54+
55+
# Message the utility that was detected based upon the file.
56+
messages.add_message(
57+
request=self.request,
58+
level=messages.INFO,
59+
message=format_html(f"<b>Utility</b>: {utility_title.title()}")
60+
)
61+
62+
# Message the count of usage events that were found in the file.
63+
found_msg = ngettext(
64+
singular=f"%d {model._meta.verbose_name.title()}",
65+
plural=f"%d {model._meta.verbose_name_plural.title()}",
66+
number=num_found,
67+
) % num_found
68+
messages.add_message(
69+
request=self.request,
70+
level=messages.INFO,
71+
message=format_html(f"<b>Found</b>: {found_msg}")
72+
)
73+
74+
# Add each usage item to database.
75+
for usage_item in usage:
76+
obj, created = model.objects.update_or_create(
77+
**usage_item,
78+
defaults=usage_item
79+
)
80+
if created:
81+
num_created += 1
82+
83+
# Message count of usage events that were created from uploaded file.
84+
create_level = messages.INFO
85+
if num_created > 0:
86+
create_level = messages.SUCCESS
87+
create_msg = ngettext(
88+
singular=f"%d {model._meta.verbose_name.title()}",
89+
plural=f"%d {model._meta.verbose_name_plural.title()}",
90+
number=num_created,
91+
) % num_created
92+
messages.add_message(
93+
request=self.request,
94+
level=create_level,
95+
message=format_html(f"<b>Created</b>: {create_msg}")
96+
)
97+
98+
return super().form_valid(form)

‎settings.example.py

+2
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@
116116
NATURAL_GAS_PREFIX = "UsageData"
117117
WATER_FILENAME = "ChartData.csv"
118118

119+
USAGE_FILE_SUFFIXES = ("csv", "xlsx")
120+
119121
WEBSITE_TITLE = "Utilities"
120122

121123
JAZZMIN_SETTINGS = {

0 commit comments

Comments
 (0)