Initial commit

.gitignore

@@ -0,0 +1,3 @@
+.venv/
+.env
+__pycache__

Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+beautifulsoup4 = "*"
+sqlalchemy = "*"
+python-dotenv = "*"
+psycopg2-binary = "*"
+apscheduler = "*"
+
+[requires]
+python_version = "3.7"

Pipfile.lock

@@ -0,0 +1,151 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "40681423ee7bb7705184a16526334130adc76c1b726852ea6797a4552c90cdac"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "apscheduler": {
+            "hashes": [
+                "sha256:529afb7909e08416132891188cbfea5351eb35e4a684b67e983d819e8d01a6b0",
+                "sha256:cde18f6dbffa1b75aff67fd7fe423a3020cb0363f6c67bd45f24306d90898231"
+            ],
+            "index": "pypi",
+            "version": "==3.6.1"
+        },
+        "beautifulsoup4": {
+            "hashes": [
+                "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
+                "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
+                "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
+            ],
+            "index": "pypi",
+            "version": "==4.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
+                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+            ],
+            "version": "==2019.6.16"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:080c72714784989474f97be9ab0ddf7b2ad2984527e77f2909fcd04d4df53809",
+                "sha256:110457be80b63ff4915febb06faa7be002b93a76e5ba19bf3f27636a2ef58598",
+                "sha256:171352a03b22fc099f15103959b52ee77d9a27e028895d7e5fde127aa8e3bac5",
+                "sha256:19d013e7b0817087517a4b3cab39c084d78898369e5c46258aab7be4f233d6a1",
+                "sha256:249b6b21ae4eb0f7b8423b330aa80fab5f821b9ffc3f7561a5e2fd6bb142cf5d",
+                "sha256:2ac0731d2d84b05c7bb39e85b7e123c3a0acd4cda631d8d542802c88deb9e87e",
+                "sha256:2b6d561193f0dc3f50acfb22dd52ea8c8dfbc64bcafe3938b5f209cc17cb6f00",
+                "sha256:2bd23e242e954214944481124755cbefe7c2cf563b1a54cd8d196d502f2578bf",
+                "sha256:3e1239242ca60b3725e65ab2f13765fc199b03af9eaf1b5572f0e97bdcee5b43",
+                "sha256:3eb70bb697abbe86b1d2b1316370c02ba320bfd1e9e35cf3b9566a855ea8e4e5",
+                "sha256:51a2fc7e94b98bd1bb5d4570936f24fc2b0541b63eccadf8fdea266db8ad2f70",
+                "sha256:52f1bdafdc764b7447e393ed39bb263eccb12bfda25a4ac06d82e3a9056251f6",
+                "sha256:5b3581319a3951f1e866f4f6c5e42023db0fae0284273b82e97dfd32c51985cd",
+                "sha256:63c1b66e3b2a3a336288e4bcec499e0dc310cd1dceaed1c46fa7419764c68877",
+                "sha256:8123a99f24ecee469e5c1339427bcdb2a33920a18bb5c0d58b7c13f3b0298ba3",
+                "sha256:85e699fcabe7f817c0f0a412d4e7c6627e00c412b418da7666ff353f38e30f67",
+                "sha256:8dbff4557bbef963697583366400822387cccf794ccb001f1f2307ed21854c68",
+                "sha256:908d21d08d6b81f1b7e056bbf40b2f77f8c499ab29e64ec5113052819ef1c89b",
+                "sha256:af39d0237b17d0a5a5f638e9dffb34013ce2b1d41441fd30283e42b22d16858a",
+                "sha256:af51bb9f055a3f4af0187149a8f60c9d516cf7d5565b3dac53358796a8fb2a5b",
+                "sha256:b2ecac57eb49e461e86c092761e6b8e1fd9654dbaaddf71a076dcc869f7014e2",
+                "sha256:cd37cc170678a4609becb26b53a2bc1edea65177be70c48dd7b39a1149cabd6e",
+                "sha256:d17e3054b17e1a6cb8c1140f76310f6ede811e75b7a9d461922d2c72973f583e",
+                "sha256:d305313c5a9695f40c46294d4315ed3a07c7d2b55e48a9010dad7db7a66c8b7f",
+                "sha256:dd0ef0eb1f7dd18a3f4187226e226a7284bda6af5671937a221766e6ef1ee88f",
+                "sha256:e1adff53b56db9905db48a972fb89370ad5736e0450b96f91bcf99cadd96cfd7",
+                "sha256:f0d43828003c82dbc9269de87aa449e9896077a71954fbbb10a614c017e65737",
+                "sha256:f78e8b487de4d92640105c1389e5b90be3496b1d75c90a666edd8737cc2dbab7"
+            ],
+            "index": "pypi",
+            "version": "==2.8.3"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:debd928b49dbc2bf68040566f55cdb3252458036464806f4094487244e2a4093",
+                "sha256:f157d71d5fec9d4bd5f51c82746b6344dffa680ee85217c123f4a0c8117c4544"
+            ],
+            "index": "pypi",
+            "version": "==0.10.3"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+            ],
+            "version": "==1.12.0"
+        },
+        "soupsieve": {
+            "hashes": [
+                "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
+                "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
+            ],
+            "version": "==1.9.3"
+        },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
+            ],
+            "index": "pypi",
+            "version": "==1.3.7"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+            ],
+            "version": "==1.25.3"
+        }
+    },
+    "develop": {}
+}

app/__init__.py

@@ -0,0 +1,35 @@
+"""Fabriek Activiteit en Groei Beheer"""
+
+import os
+import logging
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from dotenv import load_dotenv
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from app.models import Base, State, Region, Player
+
+
+load_dotenv()
+
+# database
+engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
+Session = sessionmaker(bind=engine)
+session = Session()
+
+# scheduler
+scheduler = BackgroundScheduler()
+scheduler.start()
+
+logging.basicConfig(
+    format='%(name)s - %(levelname)s - %(message)s',
+    level=logging.INFO
+)
+LOGGER = logging.getLogger(__name__)
+
+# api
+BASE_URL = os.environ["API_URL"]
+HEADERS = {
+    'Authorization': os.environ["AUTHORIZATION"]
+}

app/__main__.py

@@ -0,0 +1,58 @@
+"""Main app"""
+
+import time
+
+from app import scheduler, session, LOGGER
+from app.api import get_factories
+from app.database import get_state, save_factories
+
+
+def print_factories(factories):
+    """Print professors"""
+    for factory in factories:
+        print('{:20} {:30}'.format(
+            factory['id'],
+            factory['name'],
+        ))
+
+def job_update_factories(state_id):
+    """Update factories"""
+    LOGGER.info('Run update factories for state "%s"', state_id)
+    state = get_state(state_id)
+    LOGGER.info('"%s": get factories', state.name)
+    factories = get_factories(state.id)
+    LOGGER.info('"%s": "%s" factories', state.name, len(factories))
+    print_factories(factories)
+    # save_factories(region.id, factories)
+    LOGGER.info('"%s": done saving factories', state.name)
+
+
+def add_update_factories(state_id):
+    """Add jobs"""
+    scheduler.add_job(
+        job_update_factories,
+        'cron',
+        args=[state_id],
+        id='factories_{}'.format(state_id),
+        replace_existing=True,
+        hour='2'
+    )
+
+if __name__ == '__main__':
+    # jobs
+    # job_update_factories(2788)
+
+    # Verenigde Nederlanden
+    add_update_factories(2788)
+    # Belgium
+    add_update_factories(2604)
+    # De Provincien
+    add_update_factories(2620)
+
+    try:
+        while True:
+            time.sleep(100)
+    except KeyboardInterrupt:
+        print('Exiting application')
+        session.close()
+        exit()

app/api.py

@@ -0,0 +1,33 @@
+"""API module"""
+
+import re
+from datetime import datetime, date, timedelta
+
+import requests
+from bs4 import BeautifulSoup
+
+from app import BASE_URL, HEADERS
+
+
+def get_factories(state_id):
+    """Get factories from state"""
+    return download_factories(state_id)
+
+def download_factories(state_id):
+    """Download the players"""
+    return []
+
+def parse_factories(html):
+    """Parse html return factories"""
+    soup = BeautifulSoup(html, 'html.parser')
+    factories_tree = soup.find_all(class_='list_link')
+    factories = []
+    return factories
+
+def parse_date(date_string):
+    """Parse date to object"""
+    if 'Today' in date_string:
+        return date.today()
+    if 'Yesterday' in date_string:
+        return date.today() - timedelta(1)
+    return datetime.strptime(date_string, '%d %B %Y').date()

app/database.py

@@ -0,0 +1,15 @@
+"""Database module"""
+
+from datetime import datetime
+
+from app import session
+from app.models import State, Player
+
+
+def get_state(state_id):
+    """Get regions from state"""
+    return session.query(State).get(state_id)
+
+def save_factories(factories):
+    """Save factories to database"""
+    return

app/models.py

@@ -0,0 +1,92 @@
+"""Database models"""
+
+from sqlalchemy import Column, ForeignKey, Integer, String, \
+    DateTime, BigInteger, SmallInteger, Date
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+
+
+class Factory(Base):
+    """Model for factory"""
+    __tablename__ = 'factory'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+    player_id = Column(BigInteger, ForeignKey('player.id'))
+    player = relationship(
+        'User',
+        backref=backref('factories', lazy='dynamic')
+    )
+
+
+class FactoryTrack(Base):
+    """Model for facctory track"""
+    __tablename__ = 'factory_track'
+    id = Column(Integer, primary_key=True)
+    date_time = Column(DateTime)
+
+    state_id = Column(Integer, ForeignKey('state.id'))
+    state = relationship(
+        'State',
+        backref=backref('factory_tracks', lazy='dynamic')
+    )
+
+class FactoryLocation(Base):
+    """Model for factory location"""
+    __tablename__ = 'factory_location'
+    factory_id = Column(Integer, ForeignKey('factory.id'), primary_key=True)
+    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True)
+    from_date_time = Column(DateTime)
+    until_date_time = Column(DateTime)
+
+class FactoryStat(Base):
+    """Model for factory"""
+    __tablename__ = 'factory_stat'
+    id = Column(Integer, primary_key=True)
+    level = Column(SmallInteger)
+    workers = Column(SmallInteger)
+    experience = Column(Integer)
+    wage = Column(Integer)
+
+    factory_id = Column(Integer, ForeignKey('factory.id'))
+    factory = relationship(
+        'FactoryTrack',
+        backref=backref('factory_stats', lazy='dynamic')
+    )
+    factory_track_id = Column(Integer, ForeignKey('factory_track.id'))
+    factory_track = relationship(
+        'FactoryTrack',
+        backref=backref('factory_stats', lazy='dynamic')
+    )
+
+
+class State(Base):
+    """Model for state"""
+    __tablename__ = 'state'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    regions = relationship(
+        'Region',
+        secondary='state_region',
+        backref=backref('states', lazy='dynamic'),
+        lazy='dynamic'
+    )
+
+
+class Region(Base):
+    """Model for region"""
+    __tablename__ = 'region'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+
+class Player(Base):
+    """Model for player"""
+    __tablename__ = 'player'
+    id = Column(BigInteger, primary_key=True)
+    name = Column(String)
+    nation = Column(String)
+    registration_date = Column(Date)

example.env

@@ -0,0 +1,3 @@
+AUTHORIZATION=PLACEHOLDER
+DATABASE_URI='postgresql://hvs@localhost/hvs'
+API_URL='http://localhost:5000/api/request/'