From dba76057b3227db77394ca869de83ee87340b01b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Ho=CC=88rler?= Date: Sun, 21 Jan 2018 21:31:26 +0100 Subject: [PATCH] added new generic rss fetch and parse functions and setup to store not only today value but all values in file and update if new are available. --- .../didgeridoo/currencies/exchange_rates.py | 197 +++++++----------- .../currencies/migrations/__init__.py | 0 .../currencies/templatetags/customfilters.py | 1 + django/didgeridoo/currencies/views.py | 174 +++++++++------- .../didgeridoo/webshop/migrations/__init__.py | 0 .../webshop/templates/webshop/index.html | 2 +- 6 files changed, 184 insertions(+), 190 deletions(-) delete mode 100644 django/didgeridoo/currencies/migrations/__init__.py delete mode 100644 django/didgeridoo/webshop/migrations/__init__.py diff --git a/django/didgeridoo/currencies/exchange_rates.py b/django/didgeridoo/currencies/exchange_rates.py index 05c279f..7e7cfd9 100644 --- a/django/didgeridoo/currencies/exchange_rates.py +++ b/django/didgeridoo/currencies/exchange_rates.py @@ -11,134 +11,99 @@ Key:Value pairs of new currencys. """ -def get_exchange_rate(): - # zweitweise kann die resource nicht geladen werden. - # https://stackoverflow.com/a/43523497/4061870 - # During weekends there are no updates. - # To develop i need a testresource. - # In that case i comment the Online Resource block and uncomment the - # development Block... - - # ~~~~~~~~~~~~~~~~~~~~~ - # Online Resource block: - # ~~~~~~~~~~~~~~~~~~~~~ - today = datetime.now().strftime("%Y-%m-%d") - SNB_URL = 'https://www.snb.ch/selector/de/mmr/exfeed/rss' +def get_rss(url): urlsocket = '' try: - urlsocket = urllib.request.urlopen(SNB_URL) + urlsocket = urllib.request.urlopen(url) + return(urlsocket) except urllib.error.URLError as e: print('err: urllib.request.urlopen: ', e.reason) + + +def parse_rss(urlsocket): if urlsocket: root = ET.parse(urlsocket) - root = ET.ElementTree(root) - # ~~~~~~~~~~~~~~~~~~~~~ - # development block: - # ~~~~~~~~~~~~~~~~~~~~~ - # today = "2018-01-08" - # try: - # root = ET.ElementTree(file='rss') - # except Exception as e: - # print('exchange_rates.py_urlsocket failed %s ( - # %s) on date: %s for %s' - # % (e, type(e), root)) - # ~~~~~~~~~~~~~~~~~~~~~ + rss_tree = ET.ElementTree(root) + return(rss_tree) - # Namespaces - ns = {'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', - 'none': 'http://purl.org/rss/1.0/', - 'dc': 'http://purl.org/dc/elements/1.1/', - 'dcterms': 'http://purl.org/dc/terms/', - 'cb': 'http://www.cbwiki.net/wiki/index.php/Specification_1.2/' - } - # Pathvariables to XML Namespaces - rate_path = 'cb:statistics/cb:exchangeRate/' - observation_path = 'cb:statistics/cb:exchangeRate/cb:observation/' - exchange_rates = {} - for item in root.findall('none:item', ns): - # for eatch item n the list we grab the release date - # to evaluate if its fresh data or old: - # THE CURRENCY DATE: - datetime_str = item.find('dc:date', ns).text - # convert string to date object: - # https://stackoverflow.com/a/12282040/4061870 - # seams like snb striked the microsecond somewhere - # between Nov. and Dez. 2017 so maybe first check - # time type is with milliseconds: +def get_exchange_rate(rss_tree, ns): + # Pathvariables to XML Namespaces with + rate_path = 'cb:statistics/cb:exchangeRate/' + observation_path = 'cb:statistics/cb:exchangeRate/cb:observation/' + exchange_rates = [] + + for item in rss_tree.findall('none:item', ns): + datetime_str = item.find('dc:date', ns).text + try: + date = datetime.strptime(''.join( + datetime_str.rsplit(':', 1)), + "%Y-%m-%dT%H:%M:%S%z").strftime( + "%Y-%m-%d") + except Exception as e: + print('%s (%s)' % (e, type(e))) try: date = datetime.strptime(''.join( datetime_str.rsplit(':', 1)), - "%Y-%m-%dT%H:%M:%S%z").strftime( + "%Y-%m-%dT%H:%M:%S.%f%z").strftime( "%Y-%m-%d") + continue except Exception as e: print('%s (%s)' % (e, type(e))) - try: - date = datetime.strptime(''.join( - datetime_str.rsplit(':', 1)), - "%Y-%m-%dT%H:%M:%S.%f%z").strftime( - "%Y-%m-%d") - continue - except Exception as e: - print('%s (%s)' % (e, type(e))) - continue - # Print dates for development: - # print("date:", date, "today:", today) - # only the values of today are used so check for date in XML: - if date == today: - # now search for the currency exchange rate: - target_currency = item.find(rate_path + - 'cb:targetCurrency', ns).text - value = float(item.find(observation_path + - 'cb:value', ns).text) - value = float(value) # convert to float - foreign_value = value # copy to new value to have both. - - if item.find(observation_path + 'cb:unit_mult', ns) is None: - # because it's dangerous to check for present, - # i check for none here and have to set the target - # to 1. as im multiplying it later. - unit_mult = float("1.0") - else: - # shift left by 2 digits with "/" - # https://stackoverflow.com/questions/8362792/ - # because some currencys differ widly from CHF - unit_mult = item.find(observation_path + - 'cb:unit_mult', ns).text - # unit_mult defaults to '0' so we check for 8 decimal - # values (2..-6) they represent the fracton value to - # calculate the correct decimalpoint. - if unit_mult == '2': # thinking of Bitcoins - unit_mult = '0.01' - if unit_mult == '1': - unit_mult = '0.10' - if unit_mult == '-1': - unit_mult = '10' - if unit_mult == '-2': # Japan Yen - unit_mult = '100' - if unit_mult == '-3': - unit_mult = '1000' - if unit_mult == '-4': - unit_mult = '10000' - if unit_mult == '-5': - unit_mult = '100000' - if unit_mult == '-6': # indian rupies - unit_mult = '1000000' - unit_mult = float(unit_mult) # convert to float - # calculate the Currency to CHF: - foreign_value = 1 / value - foreign_value *= unit_mult - value = value / unit_mult - # truncate it to decimal values provided by the xml: - foreign_value_round = round(foreign_value, 5) - # Print nice setup of all calculated currencys for development: - # print("date:", date, " 1 ", target_currency, " costs: ", - # CHFvalue, "CHF and 1 ", base_currency, " costs: ", - # FOREIGNvalue_round, target_currency) - exchange_rates.update( - {target_currency: foreign_value_round}) - # Print the Dictionary: - # print(exchange_rates) - else: continue - return(exchange_rates, today) + # now search for the currency exchange rate: + target_currency = item.find(rate_path + + 'cb:targetCurrency', ns).text + value = float(item.find(observation_path + + 'cb:value', ns).text) + value = float(value) # convert to float + foreign_value = value # copy to new value to have both. + + if item.find(observation_path + 'cb:unit_mult', ns) is None: + # because it's dangerous to check for present, + # i check for none here and have to set the target + # to 1. as im multiplying it later. + unit_mult = float("1.0") + else: + # shift left by 2 digits with "/" + # https://stackoverflow.com/questions/8362792/ + # because some currencys differ widly from CHF + unit_mult = item.find(observation_path + + 'cb:unit_mult', ns).text + # unit_mult defaults to '0' so we check for 8 decimal + # values (2..-6) they represent the fracton value to + # calculate the correct decimalpoint. + if unit_mult == '2': # thinking of Bitcoins + unit_mult = '0.01' + if unit_mult == '1': + unit_mult = '0.10' + if unit_mult == '-1': + unit_mult = '10' + if unit_mult == '-2': # Japan Yen + unit_mult = '100' + if unit_mult == '-3': + unit_mult = '1000' + if unit_mult == '-4': + unit_mult = '10000' + if unit_mult == '-5': + unit_mult = '100000' + if unit_mult == '-6': # indian rupies + unit_mult = '1000000' + unit_mult = float(unit_mult) # convert to float + # calculate the Currency to CHF: + foreign_value = 1 / value + foreign_value *= unit_mult + value = value / unit_mult + # truncate it to decimal values provided by the xml: + foreign_value_round = round(foreign_value, 5) + # Print nice setup of all calculated currencys for development: + # print("date:", date, " 1 ", target_currency, " costs: ", + # CHFvalue, "CHF and 1 ", base_currency, " costs: ", + # FOREIGNvalue_round, target_currency) + data = [{'date': date, + 'currency': target_currency, + 'exchangerate': foreign_value_round}] + exchange_rates.append(data) + # Print the Dictionary: + print(exchange_rates) + return(exchange_rates) diff --git a/django/didgeridoo/currencies/migrations/__init__.py b/django/didgeridoo/currencies/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django/didgeridoo/currencies/templatetags/customfilters.py b/django/didgeridoo/currencies/templatetags/customfilters.py index 5831f92..a30e2ab 100644 --- a/django/didgeridoo/currencies/templatetags/customfilters.py +++ b/django/didgeridoo/currencies/templatetags/customfilters.py @@ -6,6 +6,7 @@ register = template.Library() @register.filter() def boldcoffee(value): + # currency_of_customer = request.session['currency'] return '%s !!gefiltert!!' % value # excample filter: {{ article.price_in_chf|boldcoffee }} diff --git a/django/didgeridoo/currencies/views.py b/django/didgeridoo/currencies/views.py index a4faae6..6da8f88 100644 --- a/django/didgeridoo/currencies/views.py +++ b/django/didgeridoo/currencies/views.py @@ -1,5 +1,5 @@ from django.shortcuts import render -import datetime +from datetime import datetime from django.views.generic.edit import UpdateView from django.core.urlresolvers import reverse_lazy from currencies.models import (ExchangeRate, @@ -25,82 +25,111 @@ def currency_update(request): def currencies(request): - # this function fetches the data from exchange_rates.py - # evaluates if the values are already stored and - # prepares the view all dynamicaly. - # It can grow in terms of more Currencies over time automaticaly. - today = '' - raw_data = [] + + """this function fetches the data from swiss national bank + evaluates if the values are already stored and + prepares a view all dynamicaly. + It can grow in terms of more Currencies over time automaticaly.""" + + # Namespaces + ns = {'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', + 'none': 'http://purl.org/rss/1.0/', + 'dc': 'http://purl.org/dc/elements/1.1/', + 'dcterms': 'http://purl.org/dc/terms/', + 'cb': 'http://www.cbwiki.net/wiki/index.php/Specification_1.2/' + } + SNB_URL = 'https://www.snb.ch/selector/de/mmr/exfeed/rss' try: - raw_data, today = exchange_rates.get_exchange_rate() + urlsocket = exchange_rates.get_rss(SNB_URL) except Exception as e: - print('views raw_data: ', raw_data, 'error:', e) # assert False - message_no = "Already querried today: " + print('currencies/views get_rss(): ', urlsocket, 'error:', e) + try: + rss_tree = exchange_rates.parse_rss(urlsocket) + except Exception as e: + print('currencies/views parse_rss(): ', rss_tree, 'error:', e) + try: + raw_data = exchange_rates.get_exchange_rate(rss_tree, ns) + except Exception as e: + print('currencies/views get_exchange_rate(): ', raw_data, 'error:', e) + + today = datetime.now().strftime("%Y-%m-%d") + + message_no = "Already querried: " + length_of_message_no = len(message_no) message_yes = " Updated successfully: " + length_of_message_yes = len(message_yes) # raw_data can be empty. In this case skip: if raw_data: - # if raw_data is not empty iterate over items in it: - for currency, rate in raw_data.items(): - # check for already existing exrates per day and add - # to message that its already been saved. - if ExchangeRate.objects.filter( - date__date=today, - name__name=currency): - message_no += currency + ", " - else: - if ExchangeRate_date.objects.filter(date=today)[:1]: - # if data and currency is not yet present, save it. - try: - # A: https://stackoverflow.com/a/27802801/4061870 - # lustigerweise gibt .values() den value und die id - # zurück. Ohne .values() gibts nur den "value" - date_dict = ExchangeRate_date.objects.filter( - date=today).values() - except Exception as e: - print('exdate_exists %s (%s) on %s' - % (e, type(e), today)) + for one_obj_of_list in raw_data: + for exchange_rate_of_one_day in one_obj_of_list: + date = exchange_rate_of_one_day['date'] + currency = exchange_rate_of_one_day['currency'] + exchangerate = exchange_rate_of_one_day['exchangerate'] + # check for already existing exrates per day and add + # to message that its already been saved. + if ExchangeRate.objects.filter( + date__date=date, + name__name=currency): + message_no += currency + ' on ' + date + ", " else: + if ExchangeRate_date.objects.filter(date=date)[:1]: + # if data and currency is not yet present, save it. + try: + # A: https://stackoverflow.com/a/27802801/4061870 + # lustigerweise gibt .values() den value und die id + # zurück. Ohne .values() gibts nur den "value" + date_dict = ExchangeRate_date.objects.filter( + date=date).values() + except Exception as e: + print('currencies/views/exdate_exists \ + %s (%s) on %s' + % (e, type(e), today)) + else: + try: + exdate = ExchangeRate_date.objects.create( + date=date) + exdate.save() + except Exception as e: + print('currencies/views/exdate_not_exists \ + %s (%s) for %s' + % (e, type(e), date)) + if ExchangeRate_name.objects.filter( + name=currency)[:1]: + # if data and currency is not yet present, save it. + try: + name_dict = ExchangeRate_name.objects.filter( + name=currency).values() + except Exception as e: + print('currencies/views/exname_exists \ + %s (%s) on %s' + % (e, type(e), currency)) + else: + try: + exname = ExchangeRate_name.objects.create( + name=currency) + exname.save() + except Exception as e: + print('currencies/views/exname_not_exists \ + %s (%s) on %s' + % (e, type(e), currency)) try: - exdate = ExchangeRate_date.objects.create( - date=today) - exdate.save() - except Exception as e: - print('exdate_not_exists %s (%s) for %s' - % (e, type(e), today)) - if ExchangeRate_name.objects.filter( - name=currency)[:1]: - # if data and currency is not yet present, save it. - try: - name_dict = ExchangeRate_name.objects.filter( - name=currency).values() - except Exception as e: - print('exname_exists %s (%s) on %s' - % (e, type(e), currency)) - else: - try: - exname = ExchangeRate_name.objects.create( - name=currency) - exname.save() - except Exception as e: - print('exname_not_exists %s (%s) on %s' - % (e, type(e), currency)) - try: - # save item to where id's match. - exrate = ExchangeRate.objects.create( - # name_id=name_id, - name_id=ExchangeRate_name.objects.get( - name=currency).id, - # date_id=date_id, - date_id=ExchangeRate_date.objects.get( - date=today).id, - exchange_rate_to_chf=rate, - ) - exrate.save() - message_yes += currency + ", " + # save item to where id's match. + exrate = ExchangeRate.objects.create( + # name_id=name_id, + name_id=ExchangeRate_name.objects.get( + name=currency).id, + # date_id=date_id, + date_id=ExchangeRate_date.objects.get( + date=date).id, + exchange_rate_to_chf=exchangerate, + ) + exrate.save() + message_yes += currency + ' on ' + date + ", " - except Exception as e: - print('exrate_create %s (%s) on %s for %s' - % (e, type(e), currency, today)) + except Exception as e: + print('currencies/views/exrate_create \ + %s (%s) on %s for %s' + % (e, type(e), currency, date)) # prepare messages: # python can not swap a char insinde a sting so i have @@ -112,11 +141,12 @@ def currencies(request): message_yes = message_yes.replace(",", "!", 1) # replace f. , with ! message_yes = message_yes[::-1] # invert the string back # here we evaluate what kind of message is valid: - if len(message_no) > 24 and len(message_yes) > 23: + if len(message_no) > length_of_message_no\ + and len(message_yes) > length_of_message_yes: message = message_no + message_yes elif len(message_no) > 24: message = message_no - elif len(message_yes) > 23: + elif len(message_yes) > 18: message = message_yes elif datetime.datetime.today().isoweekday() == 6: message = """Die Abfrage wurde ohne ergebniss beendet. @@ -130,8 +160,6 @@ def currencies(request): """ else: message = """Die Abfrage wurde ohne ergebniss beendet. - Kann es sein dass die SNB aufgrund eines Feiertages - geschlossen ist? """ # know we can query our data for presentaton: currency_list = ExchangeRate.objects.all() diff --git a/django/didgeridoo/webshop/migrations/__init__.py b/django/didgeridoo/webshop/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django/didgeridoo/webshop/templates/webshop/index.html b/django/didgeridoo/webshop/templates/webshop/index.html index ec938cb..4d13d5b 100644 --- a/django/didgeridoo/webshop/templates/webshop/index.html +++ b/django/didgeridoo/webshop/templates/webshop/index.html @@ -22,7 +22,7 @@ {{ article.category }} {{ article.stock }} - {{ article.price_in_chf|boldcoffee }} + {{ article.price_in_chf }} {% endfor %}