Initial commit

.gitignore

@@ -0,0 +1,3 @@
+.venv/
+.env
+__pycache__

Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+beautifulsoup4 = "*"
+sqlalchemy = "*"
+python-dotenv = "*"
+psycopg2-binary = "*"
+apscheduler = "*"
+
+[requires]
+python_version = "3.7"

Pipfile.lock

@@ -0,0 +1,151 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "40681423ee7bb7705184a16526334130adc76c1b726852ea6797a4552c90cdac"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "apscheduler": {
+            "hashes": [
+                "sha256:529afb7909e08416132891188cbfea5351eb35e4a684b67e983d819e8d01a6b0",
+                "sha256:cde18f6dbffa1b75aff67fd7fe423a3020cb0363f6c67bd45f24306d90898231"
+            ],
+            "index": "pypi",
+            "version": "==3.6.1"
+        },
+        "beautifulsoup4": {
+            "hashes": [
+                "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
+                "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
+                "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
+            ],
+            "index": "pypi",
+            "version": "==4.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
+                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+            ],
+            "version": "==2019.6.16"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:080c72714784989474f97be9ab0ddf7b2ad2984527e77f2909fcd04d4df53809",
+                "sha256:110457be80b63ff4915febb06faa7be002b93a76e5ba19bf3f27636a2ef58598",
+                "sha256:171352a03b22fc099f15103959b52ee77d9a27e028895d7e5fde127aa8e3bac5",
+                "sha256:19d013e7b0817087517a4b3cab39c084d78898369e5c46258aab7be4f233d6a1",
+                "sha256:249b6b21ae4eb0f7b8423b330aa80fab5f821b9ffc3f7561a5e2fd6bb142cf5d",
+                "sha256:2ac0731d2d84b05c7bb39e85b7e123c3a0acd4cda631d8d542802c88deb9e87e",
+                "sha256:2b6d561193f0dc3f50acfb22dd52ea8c8dfbc64bcafe3938b5f209cc17cb6f00",
+                "sha256:2bd23e242e954214944481124755cbefe7c2cf563b1a54cd8d196d502f2578bf",
+                "sha256:3e1239242ca60b3725e65ab2f13765fc199b03af9eaf1b5572f0e97bdcee5b43",
+                "sha256:3eb70bb697abbe86b1d2b1316370c02ba320bfd1e9e35cf3b9566a855ea8e4e5",
+                "sha256:51a2fc7e94b98bd1bb5d4570936f24fc2b0541b63eccadf8fdea266db8ad2f70",
+                "sha256:52f1bdafdc764b7447e393ed39bb263eccb12bfda25a4ac06d82e3a9056251f6",
+                "sha256:5b3581319a3951f1e866f4f6c5e42023db0fae0284273b82e97dfd32c51985cd",
+                "sha256:63c1b66e3b2a3a336288e4bcec499e0dc310cd1dceaed1c46fa7419764c68877",
+                "sha256:8123a99f24ecee469e5c1339427bcdb2a33920a18bb5c0d58b7c13f3b0298ba3",
+                "sha256:85e699fcabe7f817c0f0a412d4e7c6627e00c412b418da7666ff353f38e30f67",
+                "sha256:8dbff4557bbef963697583366400822387cccf794ccb001f1f2307ed21854c68",
+                "sha256:908d21d08d6b81f1b7e056bbf40b2f77f8c499ab29e64ec5113052819ef1c89b",
+                "sha256:af39d0237b17d0a5a5f638e9dffb34013ce2b1d41441fd30283e42b22d16858a",
+                "sha256:af51bb9f055a3f4af0187149a8f60c9d516cf7d5565b3dac53358796a8fb2a5b",
+                "sha256:b2ecac57eb49e461e86c092761e6b8e1fd9654dbaaddf71a076dcc869f7014e2",
+                "sha256:cd37cc170678a4609becb26b53a2bc1edea65177be70c48dd7b39a1149cabd6e",
+                "sha256:d17e3054b17e1a6cb8c1140f76310f6ede811e75b7a9d461922d2c72973f583e",
+                "sha256:d305313c5a9695f40c46294d4315ed3a07c7d2b55e48a9010dad7db7a66c8b7f",
+                "sha256:dd0ef0eb1f7dd18a3f4187226e226a7284bda6af5671937a221766e6ef1ee88f",
+                "sha256:e1adff53b56db9905db48a972fb89370ad5736e0450b96f91bcf99cadd96cfd7",
+                "sha256:f0d43828003c82dbc9269de87aa449e9896077a71954fbbb10a614c017e65737",
+                "sha256:f78e8b487de4d92640105c1389e5b90be3496b1d75c90a666edd8737cc2dbab7"
+            ],
+            "index": "pypi",
+            "version": "==2.8.3"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:debd928b49dbc2bf68040566f55cdb3252458036464806f4094487244e2a4093",
+                "sha256:f157d71d5fec9d4bd5f51c82746b6344dffa680ee85217c123f4a0c8117c4544"
+            ],
+            "index": "pypi",
+            "version": "==0.10.3"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+            ],
+            "version": "==1.12.0"
+        },
+        "soupsieve": {
+            "hashes": [
+                "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
+                "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
+            ],
+            "version": "==1.9.3"
+        },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
+            ],
+            "index": "pypi",
+            "version": "==1.3.7"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+            ],
+            "version": "==1.25.3"
+        }
+    },
+    "develop": {}
+}

app/__init__.py

@@ -0,0 +1,54 @@
+"""Markt Controle voor Onderzoek en Analyse"""
+
+import os
+import logging
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from dotenv import load_dotenv
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from app.models import Base, State, Region, Player
+
+
+load_dotenv()
+
+# database
+engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
+Session = sessionmaker(bind=engine)
+
+# scheduler
+scheduler = BackgroundScheduler()
+scheduler.start()
+
+# logging
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    level=logging.INFO
+)
+LOGGER = logging.getLogger(__name__)
+
+# api
+BASE_URL = os.environ["API_URL"]
+HEADERS = {
+    'Authorization': os.environ["AUTHORIZATION"]
+}
+
+# misc
+ITEMS = {
+    'oil': 3,
+    'ore': 4,
+    'uranium': 11,
+    'diamond': 15,
+    'liquid oxygen': 21,
+    'helium-3': 24,
+    'antirad': 13,
+    'spacerockets': 20,
+    'tanks': 2,
+    'aircrafts': 1,
+    'missiles': 14,
+    'bombers': 16,
+    'battleships': 18,
+    'moon tanks': 22,
+    'space stations': 23
+}

app/__main__.py

@@ -0,0 +1,51 @@
+"""Main app"""
+
+import time
+
+from app import scheduler, LOGGER
+from app.api import get_player_market #, get_state_market
+from app.database import save_market
+
+
+def print_player_market(player_market):
+    """Print player market"""
+    for item in player_market:
+        print('{:3} {:5}'.format(
+            item['id'],
+            item['price'],
+        ))
+
+def job_update_market():
+    """Update market"""
+    LOGGER.info('Get player market')
+    player_market = get_player_market()
+    LOGGER.info('Got player market')
+    print(player_market)
+
+    LOGGER.info('Get state market')
+    # state_market = get_state_market()
+    LOGGER.info('Got state market')
+
+    # print_market(factories)
+    LOGGER.info('saving markets')
+    save_market(player_market, [])
+    LOGGER.info('done saving markets')
+
+if __name__ == '__main__':
+    job_update_market()
+
+    # job
+    scheduler.add_job(
+        job_update_market,
+        'cron',
+        id='update_market',
+        replace_existing=True,
+        minute='0,10,20,30,40,50'
+    )
+
+    try:
+        while True:
+            time.sleep(100)
+    except KeyboardInterrupt:
+        LOGGER.info('Exiting application')
+        exit()

app/api.py

@@ -0,0 +1,42 @@
+"""API module"""
+
+import requests
+from bs4 import BeautifulSoup
+
+from app import BASE_URL, HEADERS, ITEMS
+
+
+def get_player_market():
+    """Get player market"""
+    # return read_player_market()
+    return download_player_market()
+
+def read_player_market():
+    """Read player_market"""
+    with open('item.html') as file:
+        return [parse_item(file)]
+
+def download_player_market():
+    """Download the market"""
+    items = []
+    for item_id in ITEMS.values():
+        print(item_id)
+        response = requests.get(
+            '{}storage/market/{}'.format(BASE_URL, item_id),
+            headers=HEADERS
+        )
+        items.append(parse_item(response.text))
+    return items
+
+def parse_item(html):
+    """Parse html return item"""
+    soup = BeautifulSoup(html, 'html.parser')
+    print(soup.find(class_='storage_buy_button'))
+    return {
+        'player_id': int(soup.find(class_='storage_buy_button')['whose']),
+        'player_name': soup.select_one('.storage_see.dot').string,
+        'price': int(soup.find(class_='storage_buy_input')['price']),
+        'amount': int(soup.find(class_='storage_market_number')['max']),
+        'total_offers': int(soup.select_one('.storage_see').string),
+        'item_type': int(soup.find(class_='storage_market_number')['url']),
+    }

app/database.py

@@ -0,0 +1,40 @@
+"""Database module"""
+
+from datetime import datetime
+
+from app import Session
+from app.models import State, Region, Player, MarketTrack, StateMarketStat, PlayerMarketStat
+
+
+def save_market(player_market, state_market):
+    """Save factories to database"""
+    session = Session()
+    session.close()
+
+    market_track = MarketTrack()
+    market_track.date_time = datetime.now()
+    session.add(market_track)
+
+    for item_dict in player_market:
+        player_market_stat = PlayerMarketStat()
+        player = session.query(Player).get(item_dict['player_id'])
+        if not player:
+            player = save_player(session, item_dict)
+        player_market_stat.player_id = player.id
+        player_market_stat.item_type = item_dict['item_type']
+        player_market_stat.total_offers = item_dict['total_offers']
+        player_market_stat.amount = item_dict['amount']
+        player_market_stat.price = item_dict['price']
+        player_market_stat.market_track_id = market_track.id
+        session.add(player_market_stat)
+
+    session.commit()
+    session.close()
+
+def save_player(session, item_dict):
+    """Save player to database"""
+    player = Player()
+    player.id = item_dict['player_id']
+    player.name = item_dict['player_name']
+    session.add(player)
+    return player

app/models.py

@@ -0,0 +1,82 @@
+"""Database models"""
+
+from sqlalchemy import Column, ForeignKey, Integer, String, \
+    DateTime, BigInteger, SmallInteger, Date
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+
+
+class State(Base):
+    """Model for state"""
+    __tablename__ = 'state'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+
+class Region(Base):
+    """Model for region"""
+    __tablename__ = 'region'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+
+class Player(Base):
+    """Model for player"""
+    __tablename__ = 'player'
+    id = Column(BigInteger, primary_key=True)
+    name = Column(String)
+    nation = Column(String)
+    registration_date = Column(Date)
+
+
+class MarketTrack(Base):
+    """Model for market track"""
+    __tablename__ = 'market_track'
+    id = Column(Integer, primary_key=True)
+    date_time = Column(DateTime)
+
+
+class PlayerMarketStat(Base):
+    """Model for market stat"""
+    __tablename__ = 'player_market_stat'
+    id = Column(Integer, primary_key=True)
+    item_type = Column(SmallInteger)
+    price = Column(Integer)
+    amount = Column(BigInteger)
+    total_offers = Column(Integer)
+
+    player_id = Column(Integer, ForeignKey('player.id'))
+    player = relationship(
+        'Player',
+        backref=backref('player_market_stats', lazy='dynamic')
+    )
+
+    market_track_id = Column(Integer, ForeignKey('market_track.id'))
+    market_track = relationship(
+        'MarketTrack',
+        backref=backref('player_market_stats', lazy='dynamic')
+    )
+
+
+class StateMarketStat(Base):
+    """Model for market stat"""
+    __tablename__ = 'state_market_stat'
+    id = Column(Integer, primary_key=True)
+    item_type = Column(SmallInteger)
+    price = Column(Integer)
+    amount = Column(BigInteger)
+
+    state_id = Column(Integer, ForeignKey('state.id'))
+    state = relationship(
+        'State',
+        backref=backref('state_market_stats', lazy='dynamic')
+    )
+
+    market_track_id = Column(Integer, ForeignKey('market_track.id'))
+    market_track = relationship(
+        'MarketTrack',
+        backref=backref('state_market_stats', lazy='dynamic')
+    )

example.env

@@ -0,0 +1,3 @@
+AUTHORIZATION=PLACEHOLDER
+DATABASE_URI='postgresql://hvs@localhost/hvs'
+API_URL='http://localhost:5000/api/request/'