Improve the way data is stored

app/__init__.py

@@ -74,7 +74,7 @@ STATE_ITEMS = {
 }
 
 MAX_OFFER = {
-    3: 204800000, 4: 204800000, 11: 15360000, 15: 153600,
+    3: 614400000, 4: 614400000, 11: 15360000, 15: 153600,
     21: 38400000, 24: 153600, 26: 614400, 13: 76800,
     20: 3840, 25: 15360000, 2: 4388571,
     1: 640000, 14: 256000, 16: 128000, 18: 128000,

app/__main__.py

@@ -16,6 +16,7 @@ if __name__ == '__main__':
         id='job_update_resource_market',
         replace_existing=True,
         minute='0,10,20,30,40,50'
+        # minute='0,5,10,15,20,25,30,35,40,45,50,55'
     )
 
     try:

app/api.py

@@ -23,13 +23,15 @@ def download_item(item_type):
 
 def download_offers(item_type):
     """Download item id"""
+    tries = 1
     html = ''
-    while not html:
+    while not html and tries <= 2:
         response = requests.get(
             '{}storage/listed/{}'.format(BASE_URL, item_type),
             headers=HEADERS
         )
         html = response.text
+        tries += 1
     return html
 
 def get_player_market():
@@ -55,7 +57,7 @@ def download_player_market():
     return resources
 
 def parse_player_item(html, item_type):
-    """Parse html return player item"""
+    """Parse html and return player"""
     soup = BeautifulSoup(html, 'html.parser')
     return {
         'player_id': int(re.sub(r'^.*\/', '', soup.select_one('.storage_see.dot')['action'])),
@@ -67,7 +69,7 @@ def parse_player_item(html, item_type):
     }
 
 def parse_player_offers(html):
-    """Parse html return player item"""
+    """Parse html and return offers"""
     soup = BeautifulSoup(html, 'html.parser')
     offers_tree = soup.find_all(class_='list_link')
     offers = []

app/app.py

@@ -8,12 +8,12 @@ def update_resource_market():
     LOGGER.info('Get player market')
     player_market = api.get_player_market()
     LOGGER.info('Got player market')
-    functions.print_offers(player_market)
+    functions.print_player_market(player_market)
 
     LOGGER.info('Get state market')
     state_market = api.get_state_market()
     LOGGER.info('Got state market')
-    print(state_market)
+    functions.print_state_market(state_market)
 
     LOGGER.info('saving markets')
     database.save_resource_market(player_market, state_market)

app/database.py

@@ -2,7 +2,7 @@
 
 from datetime import datetime
 
-from app import SESSION, functions
+from app import SESSION, functions, MAX_OFFER
 from app.models import Region, Player, MarketTrack, StateMarketStat, PlayerMarketStat
 
 
@@ -39,24 +39,35 @@ def save_player_market(market):
 def _save_player_market(session, market_track, market):
     """Save player market to database"""
     for item_type, offers in market.items():
-        if offers:
-            item_dict = offers[0]
-            market_stat = PlayerMarketStat()
-            player = session.query(Player).get(item_dict['player_id'])
-            if not player:
-                player = save_player(session, item_dict)
-            market_stat.player_id = player.id
-            market_stat.item_type = item_type
-            market_stat.amount = item_dict['amount']
-            market_stat.price = item_dict['price']
+        if not offers:
+            continue
+        item_dict = offers[0]
+        market_stat = PlayerMarketStat()
+        player = session.query(Player).get(item_dict['player_id'])
+        if not player:
+            player = save_player(session, item_dict)
+        market_stat.player_id = player.id
+        market_stat.item_type = item_type
+        market_stat.amount = item_dict['amount']
+        market_stat.price = item_dict['price']
+
+        market_stat.total_offers = len(offers)
+        one_t_average = functions.calculate_purchage_amount(offers, 1e12) / 100
+        base_average = 3 * MAX_OFFER[item_type] * one_t_average
+        market_stat.base_average = base_average
+        market_stat.one_average = functions.calculate_purchage_amount(offers, 1 * base_average)
+        market_stat.two_average = functions.calculate_purchage_amount(offers, 2 * base_average)
+        market_stat.three_average = functions.calculate_purchage_amount(offers, 3 * base_average)
+        market_stat.four_average = functions.calculate_purchage_amount(offers, 4 * base_average)
+        market_stat.five_average = functions.calculate_purchage_amount(offers, 5 * base_average)
+        market_stat.six_average = functions.calculate_purchage_amount(offers, 6 * base_average)
+        market_stat.seven_average = functions.calculate_purchage_amount(offers, 7 * base_average)
+        market_stat.eight_average = functions.calculate_purchage_amount(offers, 8 * base_average)
+        market_stat.nine_average = functions.calculate_purchage_amount(offers, 9 * base_average)
+        market_stat.ten_average = functions.calculate_purchage_amount(offers, 10 * base_average)
 
-            market_stat.total_offers = len(offers)
-            market_stat.half_t_average = functions.calculate_purchage_amount(offers, 5e11)
-            market_stat.one_t_average = functions.calculate_purchage_amount(offers, 1e12)
-            market_stat.two_t_average = functions.calculate_purchage_amount(offers, 2e12)
-            market_stat.five_t_average = functions.calculate_purchage_amount(offers, 5e12)
-            market_stat.market_track_id = market_track.id
-            session.add(market_stat)
+        market_stat.market_track_id = market_track.id
+        session.add(market_stat)
 
 def _save_state_market(session, market_track, market):
     """Save state market"""

app/functions.py

@@ -3,19 +3,50 @@
 import math
 
 
-def print_offers(market):
-    """Print offers"""
-    print('id     lowest       0.5T         1T         2T         5T')
+from app import MAX_OFFER
+
+
+def print_player_market(market):
+    """Print player market"""
+    print('id     lowest         1T         2T         3T         4T         5T')
     for resource_type, offers in market.items():
-        print('{:2} {:10.2f} {:10.2f} {:10.2f} {:10.2f} {:10.2f}'.format(
+        print('{:2} {:10.2f} {:10.2f} {:10.2f} {:10.2f} {:10.2f} {:10.2f}'.format(
             resource_type,
             offers[0]['price'] / 100,
-            calculate_purchage_amount(offers, 5e11) / 100,
             calculate_purchage_amount(offers, 1e12) / 100,
             calculate_purchage_amount(offers, 2e12) / 100,
+            calculate_purchage_amount(offers, 3e12) / 100,
+            calculate_purchage_amount(offers, 4e12) / 100,
             calculate_purchage_amount(offers, 5e12) / 100,
         ).replace(',', '.'))
 
+    for resource_type, offers in market.items():
+        max_offer = MAX_OFFER[resource_type]
+        prices = str(resource_type)
+        one_t_average = calculate_purchage_amount(offers, 1e12) / 100
+        for i in range(1, 20):
+            prices += ',{}'.format(
+                calculate_purchage_amount(offers, i * 3 * max_offer * one_t_average) / 100,
+            )
+        print(prices)
+
+def print_state_market(market):
+    """Print state offers"""
+    print('  id  region_id region_name')
+    for item in market:
+        print('{:6} {:8} {:20} {:14.2f} {:10}'.format(
+            item['item_type'],
+            item['region_id'],
+            item['region_name'],
+            item['price'] / 100,
+            item['amount']
+        ).replace(',', '.'))
+
+def calculate_average_price(offers, amount):
+    """Calculate average price based on amount"""
+    total = calculate_price(offers, amount)
+    return total / amount
+
 def calculate_price(offers, amount):
     """Calculate price for amount"""
     tmp_amount = amount
@@ -34,15 +65,17 @@ def calculate_purchage_amount(offers, money):
     """Calculate purchage amount"""
     tmp_money = money * 100
     total_amount = 0
+    spend_money = 0
     for offer in offers:
         buy_amount = math.floor(tmp_money / (offer['price']))
         if buy_amount > 0:
             if buy_amount > offer['amount']:
                 buy_amount = offer['amount']
-            tmp_money -= buy_amount * (offer['price'])
+            tmp_money -= buy_amount * offer['price']
+            spend_money += buy_amount * offer['price']
             total_amount += buy_amount
             if tmp_money == 0:
                 break
         else:
             break
-    return round(money * 100 / total_amount)
+    return round(spend_money / total_amount)

app/jobs.py

@@ -5,4 +5,3 @@ from app import app
 def update_resource_market():
     """job for update resource market"""
     app.update_resource_market()
-

app/models.py

@@ -49,10 +49,17 @@ class PlayerMarketStat(Base):
     item_type = Column(SmallInteger)
     price = Column(Integer)
     amount = Column(BigInteger)
-    half_t_average = Column(Integer)
-    one_t_average = Column(Integer)
-    two_t_average = Column(Integer)
-    five_t_average = Column(Integer)
+    base_average = Column(BigInteger)
+    one_average = Column(Integer)
+    two_average = Column(Integer)
+    three_average = Column(Integer)
+    four_average = Column(Integer)
+    five_average = Column(Integer)
+    six_average = Column(Integer)
+    seven_average = Column(Integer)
+    eight_average = Column(Integer)
+    nine_average = Column(Integer)
+    ten_average = Column(Integer)
     total_offers = Column(Integer)
 
     player_id = Column(BigInteger, ForeignKey('player.id'))