A lot of improvements

.gitignore

@@ -2,3 +2,4 @@
 .env
 __pycache__
 *.html
+*.log

app/__init__.py

@@ -25,12 +25,30 @@ SCHEDULER = BackgroundScheduler(
 )
 SCHEDULER.start()
 
-# logging
-logging.basicConfig(
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-    level=logging.INFO
-)
-LOGGER = logging.getLogger(__name__)
+# get logger
+LOGGER = logging.getLogger('mcoa')
+LOGGER.setLevel(logging.DEBUG)
+SCHEDULER_LOGGER = logging.getLogger('apscheduler')
+SCHEDULER_LOGGER.setLevel(logging.DEBUG)
+
+# create file handler
+FILE_HANDLER = logging.FileHandler('output.log')
+FILE_HANDLER.setLevel(logging.DEBUG)
+
+# create console handler
+STREAM_HANDLER = logging.StreamHandler()
+STREAM_HANDLER.setLevel(logging.INFO)
+
+# create formatter and add it to the handlers
+FORMATTER = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+STREAM_HANDLER.setFormatter(FORMATTER)
+FILE_HANDLER.setFormatter(FORMATTER)
+
+# add the handlers to logger
+LOGGER.addHandler(STREAM_HANDLER)
+LOGGER.addHandler(FILE_HANDLER)
+SCHEDULER_LOGGER.addHandler(STREAM_HANDLER)
+SCHEDULER_LOGGER.addHandler(FILE_HANDLER)
 
 # api
 BASE_URL = os.environ["API_URL"]

app/__main__.py

@@ -1,34 +1,17 @@
 """Main app"""
 
+import sys
 import time
 
-from app import SCHEDULER, LOGGER
-from app.api import get_player_market, get_state_market
-from app.database import save_resource_market
+from app import SCHEDULER, LOGGER, jobs
 
 
-def job_update_resource_market():
-    """Update market"""
-    LOGGER.info('Get player market')
-    player_market = get_player_market()
-    LOGGER.info('Got player market')
-    # print_offers(player_market)
-
-    LOGGER.info('Get state market')
-    state_market = get_state_market()
-    LOGGER.info('Got state market')
-    # print(state_market)
-
-    LOGGER.info('saving markets')
-    save_resource_market(player_market, state_market)
-    LOGGER.info('done saving markets')
-
 if __name__ == '__main__':
-    # job_update_resource_market()
+    jobs.update_resource_market()
 
-    # job
+    LOGGER.info('Starting application')
     SCHEDULER.add_job(
-        job_update_resource_market,
+        jobs.update_resource_market,
         'cron',
         id='job_update_resource_market',
         replace_existing=True,
@@ -41,4 +24,4 @@ if __name__ == '__main__':
     except KeyboardInterrupt:
         LOGGER.info('Exiting application')
         SCHEDULER.shutdown()
-        exit()
+        sys.exit()

app/api.py

@@ -5,18 +5,20 @@ import re
 import requests
 from bs4 import BeautifulSoup
 
-from app import BASE_URL, HEADERS, RESOURCES, ITEMS, STATE_ITEMS
+from app import BASE_URL, HEADERS, RESOURCES, STATE_ITEMS
 
 
 def download_item(item_type):
     """Download item id"""
+    tries = 1
     html = ''
-    while not html:
-        response = requests.get(
+    while not html and tries <= 2:
+        response = requests.post(
             '{}storage/market/{}'.format(BASE_URL, item_type),
             headers=HEADERS
         )
         html = response.text
+        tries += 1
     return html
 
 def download_offers(item_type):
@@ -71,7 +73,9 @@ def parse_player_offers(html):
     offers = []
     for offer_tree in offers_tree:
         offers.append({
-            'player_id': int(re.sub(r'^.*\/', '', offer_tree.select_one('.results_date')['action'])),
+            'player_id': int(
+                re.sub(r'^.*\/', '', offer_tree.select_one('.results_date')['action'])
+            ),
             'player_name': offer_tree.select_one('.results_date').string,
             'price': int(float(offer_tree.select('.list_level')[1]['rat'])*100),
             'amount': int(offer_tree.select_one('.list_level.imp.small')['rat']),
@@ -93,7 +97,8 @@ def download_state_market():
     items = []
     for item_type in STATE_ITEMS.values():
         item = download_item(item_type)
-        items.append(parse_state_item(item, item_type))
+        if item:
+            items.append(parse_state_item(item, item_type))
     return items
 
 def parse_state_item(html, item_type):

app/app.py

@@ -1,49 +1,20 @@
 """general methods"""
 
-import math
+from app import LOGGER, api, database, functions
 
-from app import MAX_OFFER
 
-def print_offers(market):
-    """Print offers"""
-    print('id     lowest       0.5T         1T         2T         5T')
-    for resource_type, offers in market.items():
-        print('{:2} {:10.2f} {:10.2f} {:10.2f} {:10.2f} {:10.2f}'.format(
-            resource_type,
-            offers[0]['price'] / 100,
-            calculate_purchage_amount(offers, 5e11) / 100,
-            calculate_purchage_amount(offers, 1e12) / 100,
-            calculate_purchage_amount(offers, 2e12) / 100,
-            calculate_purchage_amount(offers, 5e12) / 100,
-        ).replace(',', '.'))
+def update_resource_market():
+    """update resource market"""
+    LOGGER.info('Get player market')
+    player_market = api.get_player_market()
+    LOGGER.info('Got player market')
+    functions.print_offers(player_market)
 
-def calculate_price(offers, amount):
-    """Calculate price for amount"""
-    tmp_amount = amount
-    total_price = 0
-    for offer in offers:
-        buy_amount = offer['amount']
-        if buy_amount > tmp_amount:
-            buy_amount = tmp_amount
-        tmp_amount -= buy_amount
-        total_price += buy_amount * offer['price']
-        if tmp_amount == 0:
-            break
-    return total_price
+    LOGGER.info('Get state market')
+    state_market = api.get_state_market()
+    LOGGER.info('Got state market')
+    print(state_market)
 
-def calculate_purchage_amount(offers, money):
-    """Calculate purchage amount"""
-    tmp_money = money * 100
-    total_amount = 0
-    for offer in offers:
-        buy_amount = math.floor(tmp_money / (offer['price']))
-        if buy_amount > 0:
-            if buy_amount > offer['amount']:
-                buy_amount = offer['amount']
-            tmp_money -= buy_amount * (offer['price'])
-            total_amount += buy_amount
-            if tmp_money == 0:
-                break
-        else:
-            break
-    return round(money * 100 / total_amount)
+    LOGGER.info('saving markets')
+    database.save_resource_market(player_market, state_market)
+    LOGGER.info('done saving markets')

app/database.py

@@ -2,9 +2,8 @@
 
 from datetime import datetime
 
-from app import SESSION
-from app.models import State, Region, Player, MarketTrack, StateMarketStat, PlayerMarketStat
-from app.app import calculate_purchage_amount
+from app import SESSION, functions
+from app.models import Region, Player, MarketTrack, StateMarketStat, PlayerMarketStat
 
 
 def get_new_market_track(session, resources, state_resources, items):
@@ -52,10 +51,10 @@ def _save_player_market(session, market_track, market):
             market_stat.price = item_dict['price']
 
             market_stat.total_offers = len(offers)
-            market_stat.half_t_average = calculate_purchage_amount(offers, 5e11)
-            market_stat.one_t_average = calculate_purchage_amount(offers, 1e12)
-            market_stat.two_t_average = calculate_purchage_amount(offers, 2e12)
-            market_stat.five_t_average = calculate_purchage_amount(offers, 5e12)
+            market_stat.half_t_average = functions.calculate_purchage_amount(offers, 5e11)
+            market_stat.one_t_average = functions.calculate_purchage_amount(offers, 1e12)
+            market_stat.two_t_average = functions.calculate_purchage_amount(offers, 2e12)
+            market_stat.five_t_average = functions.calculate_purchage_amount(offers, 5e12)
             market_stat.market_track_id = market_track.id
             session.add(market_stat)
 

app/functions.py

@@ -0,0 +1,48 @@
+"""General functions"""
+
+import math
+
+
+def print_offers(market):
+    """Print offers"""
+    print('id     lowest       0.5T         1T         2T         5T')
+    for resource_type, offers in market.items():
+        print('{:2} {:10.2f} {:10.2f} {:10.2f} {:10.2f} {:10.2f}'.format(
+            resource_type,
+            offers[0]['price'] / 100,
+            calculate_purchage_amount(offers, 5e11) / 100,
+            calculate_purchage_amount(offers, 1e12) / 100,
+            calculate_purchage_amount(offers, 2e12) / 100,
+            calculate_purchage_amount(offers, 5e12) / 100,
+        ).replace(',', '.'))
+
+def calculate_price(offers, amount):
+    """Calculate price for amount"""
+    tmp_amount = amount
+    total_price = 0
+    for offer in offers:
+        buy_amount = offer['amount']
+        if buy_amount > tmp_amount:
+            buy_amount = tmp_amount
+        tmp_amount -= buy_amount
+        total_price += buy_amount * offer['price']
+        if tmp_amount == 0:
+            break
+    return total_price
+
+def calculate_purchage_amount(offers, money):
+    """Calculate purchage amount"""
+    tmp_money = money * 100
+    total_amount = 0
+    for offer in offers:
+        buy_amount = math.floor(tmp_money / (offer['price']))
+        if buy_amount > 0:
+            if buy_amount > offer['amount']:
+                buy_amount = offer['amount']
+            tmp_money -= buy_amount * (offer['price'])
+            total_amount += buy_amount
+            if tmp_money == 0:
+                break
+        else:
+            break
+    return round(money * 100 / total_amount)

app/jobs.py

@@ -0,0 +1,8 @@
+"""Jobs for scheduler module"""
+
+from app import app
+
+def update_resource_market():
+    """job for update resource market"""
+    app.update_resource_market()
+