Working on application

app/__init__.py

@@ -14,16 +14,15 @@ from app.models import Base, State, Region, Player
 load_dotenv()
 
 # database
-engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
-Session = sessionmaker(bind=engine)
+ENGINE = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
+SESSION = sessionmaker(bind=ENGINE)
 
 # scheduler
-scheduler = BackgroundScheduler(
+SCHEDULER = BackgroundScheduler(
     daemon=True,
     job_defaults={'misfire_grace_time': 5*60},
 )
-
-scheduler.start()
+SCHEDULER.start()
 
 # logging
 logging.basicConfig(
@@ -39,20 +38,26 @@ HEADERS = {
 }
 
 # misc
+RESOURCES = {
+    'oil': 3, 'ore': 4, 'uranium': 11, 'diamond': 15,
+}
+
 ITEMS = {
-    'oil': 3,
-    'ore': 4,
-    'uranium': 11,
-    'diamond': 15,
-    'liquid oxygen': 21,
-    'helium-3': 24,
-    'antirad': 13,
-    'spacerockets': 20,
-    'tanks': 2,
-    'aircrafts': 1,
-    'missiles': 14,
-    'bombers': 16,
-    'battleships': 18,
-    'moon tanks': 22,
-    'space stations': 23
+    'liquid oxygen': 21, 'helium-3': 24, 'rivalium': 26, 'antirad': 13,
+    'spacerockets': 20, 'lss': 25, 'tanks': 2,
+    'aircrafts': 1, 'missiles': 14, 'bombers': 16, 'battleships': 18,
+    'laser drones': 27, 'moon tanks': 22, 'space stations': 23
+}
+
+STATE_ITEMS = {
+    'gold': 1000, 'oil': 1003, 'ore': 1004, 'uranium': 1011,
+    'diamond': 1015,
+}
+
+MAX_OFFER = {
+    3: 204800000, 4: 204800000, 11: 15360000, 15: 153600,
+    21: 38400000, 24: 153600, 26: 614400, 13: 76800,
+    20: 3840, 25: 15360000, 2: 4388571,
+    1: 640000, 14: 256000, 16: 128000, 18: 128000,
+    27: 256000, 22: 12800, 23: 1280
 }

app/__main__.py

@@ -2,9 +2,10 @@
 
 import time
 
-from app import scheduler, LOGGER
-from app.api import get_player_market #, get_state_market
-from app.database import save_market
+from app import SCHEDULER, LOGGER
+from app.app import print_offers
+from app.api import get_player_market, get_state_market
+from app.database import save_resource_market
 
 
 def print_player_market(player_market):
@@ -15,30 +16,30 @@ def print_player_market(player_market):
             item['price'],
         ))
 
-def job_update_market():
+def job_update_resource_market():
     """Update market"""
     LOGGER.info('Get player market')
     player_market = get_player_market()
     LOGGER.info('Got player market')
-    print(player_market)
+    print_offers(player_market)
 
     LOGGER.info('Get state market')
-    # state_market = get_state_market()
+    state_market = get_state_market()
     LOGGER.info('Got state market')
+    # print(state_market)
 
-    # print_market(factories)
     LOGGER.info('saving markets')
-    save_market(player_market, [])
+    save_resource_market(player_market, state_market)
     LOGGER.info('done saving markets')
 
 if __name__ == '__main__':
-    job_update_market()
+    job_update_resource_market()
 
     # job
-    scheduler.add_job(
-        job_update_market,
+    SCHEDULER.add_job(
+        job_update_resource_market,
         'cron',
-        id='update_market',
+        id='job_update_resource_market',
         replace_existing=True,
         minute='0,10,20,30,40,50'
     )
@@ -48,5 +49,5 @@ if __name__ == '__main__':
             time.sleep(100)
     except KeyboardInterrupt:
         LOGGER.info('Exiting application')
-        scheduler.shutdown()
+        SCHEDULER.shutdown()
         exit()

app/api.py

@@ -5,38 +5,104 @@ import re
 import requests
 from bs4 import BeautifulSoup
 
-from app import BASE_URL, HEADERS, ITEMS
+from app import BASE_URL, HEADERS, RESOURCES, ITEMS, STATE_ITEMS
 
 
+def download_item(item_type):
+    """Download item id"""
+    html = ''
+    while not html:
+        response = requests.get(
+            '{}storage/market/{}'.format(BASE_URL, item_type),
+            headers=HEADERS
+        )
+        html = response.text
+    return html
+
+def download_offers(item_type):
+    """Download item id"""
+    html = ''
+    while not html:
+        response = requests.get(
+            '{}storage/listed/{}'.format(BASE_URL, item_type),
+            headers=HEADERS
+        )
+        html = response.text
+    return html
+
 def get_player_market():
     """Get player market"""
     # return read_player_market()
     return download_player_market()
 
 def read_player_market():
-    """Read player_market"""
-    with open('item.html') as file:
-        return [parse_item(file)]
+    """Read player market"""
+    with open('items.html') as file:
+        resources = {}
+        for resource_type in RESOURCES.values():
+            resources[resource_type] = parse_player_offers(file)
+            break
+        return resources
 
 def download_player_market():
     """Download the market"""
-    items = []
-    for item_id in ITEMS.values():
-        response = requests.get(
-            '{}storage/market/{}'.format(BASE_URL, item_id),
-            headers=HEADERS
-        )
-        items.append(parse_item(response.text))
-    return items
+    resources = {}
+    for resource_type in RESOURCES.values():
+        html = download_offers(resource_type)
+        resources[resource_type] = parse_player_offers(html)
+    return resources
 
-def parse_item(html):
-    """Parse html return item"""
+def parse_player_item(html, item_type):
+    """Parse html return player item"""
     soup = BeautifulSoup(html, 'html.parser')
     return {
         'player_id': int(re.sub(r'^.*\/', '', soup.select_one('.storage_see.dot')['action'])),
         'player_name': soup.select_one('.storage_see.dot').string,
-        'price': int(re.sub(r'\..*$', '', soup.find(class_='storage_buy_input')['price'])),
+        'price': int(float(soup.find(class_='storage_buy_input')['price'])*100),
         'amount': int(soup.find(class_='storage_market_number')['max']),
         'total_offers': int(re.sub(r'\..*$', '', soup.select_one('.storage_see').string)),
-        'item_type': int(soup.find(class_='storage_market_number')['url']),
+        'item_type': item_type,
+    }
+
+def parse_player_offers(html):
+    """Parse html return player item"""
+    soup = BeautifulSoup(html, 'html.parser')
+    offers_tree = soup.find_all(class_='list_link')
+    offers = []
+    for offer_tree in offers_tree:
+        offers.append({
+            'player_id': int(re.sub(r'^.*\/', '', offer_tree.select_one('.results_date')['action'])),
+            'player_name': offer_tree.select_one('.results_date').string,
+            'price': int(float(offer_tree.select('.list_level')[1]['rat'])*100),
+            'amount': int(offer_tree.select_one('.list_level.imp.small')['rat']),
+        })
+    return offers
+
+def get_state_market():
+    """Get state market"""
+    # return read_state_market()
+    return download_state_market()
+
+def read_state_market():
+    """Read state market"""
+    with open('state_item.html') as file:
+        return [parse_state_item(file, 1001)]
+
+def download_state_market():
+    """Download the state market"""
+    items = []
+    for item_type in STATE_ITEMS.values():
+        item = download_item(item_type)
+        items.append(parse_state_item(item, item_type))
+    return items
+
+def parse_state_item(html, item_type):
+    """Parse htm return state item"""
+    soup = BeautifulSoup(html, 'html.parser')
+    return {
+        'region_id': int(re.sub(r'^.*\/', '', soup.select_one('.storage_see.dot')['action'])),
+        'region_name': soup.select_one('.storage_see.dot').string,
+        'price': int(float(soup.find(class_='storage_buy_input')['price'])*100),
+        'amount': int(soup.find(class_='storage_market_number')['max']),
+        'item_type': item_type,
     }

app/app.py

@@ -0,0 +1,50 @@
+"""general methods"""
+
+import math
+
+from app import MAX_OFFER
+
+def print_offers(market):
+    """Print offers"""
+    for resource_type, offers in market.items():
+        purchage_money = 2e12
+        purchage_average = calculate_purchage_amount(offers, purchage_money)
+        money = MAX_OFFER[resource_type] * 5
+        price = calculate_price(offers, money)
+        print('{:2} {:15,} {:25,} {:15,}'.format(
+            resource_type, 
+            purchage_average / 100,
+            price,
+            offers[0]['price'] / 100,
+        ).replace(',', '.'))
+
+def calculate_price(offers, amount):
+    """Calculate price for amount"""
+    tmp_amount = amount
+    total_price = 0
+    for offer in offers:
+        buy_amount = offer['amount'] 
+        if buy_amount > tmp_amount:
+            buy_amount = tmp_amount
+        tmp_amount -= buy_amount
+        total_price += buy_amount * offer['price']
+        if tmp_amount == 0:
+            break
+    return total_price
+
+def calculate_purchage_amount(offers, money):
+    """Calculate purchage amount"""
+    tmp_money = money * 100
+    total_amount = 0
+    for offer in offers:
+        buy_amount = math.floor(tmp_money / (offer['price']))
+        if buy_amount > 0:
+            if buy_amount > offer['amount']:
+                buy_amount = offer['amount']
+            tmp_money -= buy_amount * (offer['price'])
+            total_amount += buy_amount
+            if tmp_money == 0:
+                break
+        else:
+            break
+    return round(money * 100 / total_amount)

app/database.py

@@ -2,34 +2,61 @@
 
 from datetime import datetime
 
-from app import Session
+from app import SESSION
 from app.models import State, Region, Player, MarketTrack, StateMarketStat, PlayerMarketStat
 
 
-def save_market(player_market, state_market):
+def get_new_market_track(session):
+    """Get ner market track"""
+    market_track = MarketTrack()
+    market_track.date_time = datetime.now()
+    session.add(market_track)
+    return market_track
+
+def save_resource_market(player_market, state_market):
     """Save factories to database"""
-    session = Session()
+    session = SESSION()
+
+    market_track = get_new_market_track(session)
+    market_track.player_resources = True
+    market_track.state_resources = True
+    _save_player_market(session, market_track, player_market)
+    _save_state_market(session, market_track, state_market)
+
+    # session.commit()
     session.close()
 
-    market_track = MarketTrack()
-    market_track.date_time = datetime.now()
+def save_player_market(market):
+    """Save player market"""
+    session = SESSION()
+    market_track = get_new_market_track(session)
     session.add(market_track)
+    _save_player_market(session, market_track, market)
 
-    for item_dict in player_market:
-        player_market_stat = PlayerMarketStat()
+def _save_player_market(session, market_track, market):
+    """Save player market to database"""
+    for item_type, offers in market.items():
+        item_dict = offers[0]
+        market_stat = PlayerMarketStat()
         player = session.query(Player).get(item_dict['player_id'])
         if not player:
             player = save_player(session, item_dict)
-        player_market_stat.player_id = player.id
-        player_market_stat.item_type = item_dict['item_type']
-        player_market_stat.total_offers = item_dict['total_offers']
-        player_market_stat.amount = item_dict['amount']
-        player_market_stat.price = item_dict['price']
-        player_market_stat.market_track_id = market_track.id
-        session.add(player_market_stat)
-
-    session.commit()
-    session.close()
+        market_stat.player_id = player.id
+        market_stat.item_type = item_type
+        market_stat.total_offers = len(offers)
+        market_stat.amount = item_dict['amount']
+        market_stat.price = item_dict['price']
+        market_stat.market_track_id = market_track.id
+        session.add(market_stat)
+
+def _save_state_market(session, market_track, market):
+    """Save state market"""
+    for item_dict in market:
+        print(item_dict)
+        market_stat = StateMarketStat()
+        market_stat.market_track_id = market_track.id
+        session.add(market_stat)
+
 
 def save_player(session, item_dict):
     """Save player to database"""

app/models.py

@@ -1,7 +1,7 @@
 """Database models"""
 
 from sqlalchemy import Column, ForeignKey, Integer, String, \
-    DateTime, BigInteger, SmallInteger, Date
+    DateTime, BigInteger, SmallInteger, Date, Boolean
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -37,6 +37,9 @@ class MarketTrack(Base):
     __tablename__ = 'market_track'
     id = Column(Integer, primary_key=True)
     date_time = Column(DateTime)
+    player_resources = Column(Boolean, server_default='f', default=False)
+    state_resources = Column(Boolean, server_default='f', default=False)
+    items = Column(Boolean, server_default='f', default=False)
 
 
 class PlayerMarketStat(Base):
@@ -46,6 +49,10 @@ class PlayerMarketStat(Base):
     item_type = Column(SmallInteger)
     price = Column(Integer)
     amount = Column(BigInteger)
+    half_t_average = Column(Integer)
+    one_t_average = Column(Integer)
+    two_t_average = Column(Integer)
+    five_t_average = Column(Integer)
     total_offers = Column(Integer)
 
     player_id = Column(Integer, ForeignKey('player.id'))