Initial commit

.gitignore

@@ -0,0 +1,3 @@
+.venv/
+.env
+__pycache__

Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+beautifulsoup4 = "*"
+sqlalchemy = "*"
+python-dotenv = "*"
+psycopg2-binary = "*"
+apscheduler = "*"
+
+[requires]
+python_version = "3.7"

Pipfile.lock

@@ -0,0 +1,151 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "40681423ee7bb7705184a16526334130adc76c1b726852ea6797a4552c90cdac"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "apscheduler": {
+            "hashes": [
+                "sha256:529afb7909e08416132891188cbfea5351eb35e4a684b67e983d819e8d01a6b0",
+                "sha256:cde18f6dbffa1b75aff67fd7fe423a3020cb0363f6c67bd45f24306d90898231"
+            ],
+            "index": "pypi",
+            "version": "==3.6.1"
+        },
+        "beautifulsoup4": {
+            "hashes": [
+                "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
+                "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
+                "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
+            ],
+            "index": "pypi",
+            "version": "==4.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
+                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+            ],
+            "version": "==2019.6.16"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:080c72714784989474f97be9ab0ddf7b2ad2984527e77f2909fcd04d4df53809",
+                "sha256:110457be80b63ff4915febb06faa7be002b93a76e5ba19bf3f27636a2ef58598",
+                "sha256:171352a03b22fc099f15103959b52ee77d9a27e028895d7e5fde127aa8e3bac5",
+                "sha256:19d013e7b0817087517a4b3cab39c084d78898369e5c46258aab7be4f233d6a1",
+                "sha256:249b6b21ae4eb0f7b8423b330aa80fab5f821b9ffc3f7561a5e2fd6bb142cf5d",
+                "sha256:2ac0731d2d84b05c7bb39e85b7e123c3a0acd4cda631d8d542802c88deb9e87e",
+                "sha256:2b6d561193f0dc3f50acfb22dd52ea8c8dfbc64bcafe3938b5f209cc17cb6f00",
+                "sha256:2bd23e242e954214944481124755cbefe7c2cf563b1a54cd8d196d502f2578bf",
+                "sha256:3e1239242ca60b3725e65ab2f13765fc199b03af9eaf1b5572f0e97bdcee5b43",
+                "sha256:3eb70bb697abbe86b1d2b1316370c02ba320bfd1e9e35cf3b9566a855ea8e4e5",
+                "sha256:51a2fc7e94b98bd1bb5d4570936f24fc2b0541b63eccadf8fdea266db8ad2f70",
+                "sha256:52f1bdafdc764b7447e393ed39bb263eccb12bfda25a4ac06d82e3a9056251f6",
+                "sha256:5b3581319a3951f1e866f4f6c5e42023db0fae0284273b82e97dfd32c51985cd",
+                "sha256:63c1b66e3b2a3a336288e4bcec499e0dc310cd1dceaed1c46fa7419764c68877",
+                "sha256:8123a99f24ecee469e5c1339427bcdb2a33920a18bb5c0d58b7c13f3b0298ba3",
+                "sha256:85e699fcabe7f817c0f0a412d4e7c6627e00c412b418da7666ff353f38e30f67",
+                "sha256:8dbff4557bbef963697583366400822387cccf794ccb001f1f2307ed21854c68",
+                "sha256:908d21d08d6b81f1b7e056bbf40b2f77f8c499ab29e64ec5113052819ef1c89b",
+                "sha256:af39d0237b17d0a5a5f638e9dffb34013ce2b1d41441fd30283e42b22d16858a",
+                "sha256:af51bb9f055a3f4af0187149a8f60c9d516cf7d5565b3dac53358796a8fb2a5b",
+                "sha256:b2ecac57eb49e461e86c092761e6b8e1fd9654dbaaddf71a076dcc869f7014e2",
+                "sha256:cd37cc170678a4609becb26b53a2bc1edea65177be70c48dd7b39a1149cabd6e",
+                "sha256:d17e3054b17e1a6cb8c1140f76310f6ede811e75b7a9d461922d2c72973f583e",
+                "sha256:d305313c5a9695f40c46294d4315ed3a07c7d2b55e48a9010dad7db7a66c8b7f",
+                "sha256:dd0ef0eb1f7dd18a3f4187226e226a7284bda6af5671937a221766e6ef1ee88f",
+                "sha256:e1adff53b56db9905db48a972fb89370ad5736e0450b96f91bcf99cadd96cfd7",
+                "sha256:f0d43828003c82dbc9269de87aa449e9896077a71954fbbb10a614c017e65737",
+                "sha256:f78e8b487de4d92640105c1389e5b90be3496b1d75c90a666edd8737cc2dbab7"
+            ],
+            "index": "pypi",
+            "version": "==2.8.3"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:debd928b49dbc2bf68040566f55cdb3252458036464806f4094487244e2a4093",
+                "sha256:f157d71d5fec9d4bd5f51c82746b6344dffa680ee85217c123f4a0c8117c4544"
+            ],
+            "index": "pypi",
+            "version": "==0.10.3"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+            ],
+            "version": "==1.12.0"
+        },
+        "soupsieve": {
+            "hashes": [
+                "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
+                "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
+            ],
+            "version": "==1.9.3"
+        },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
+            ],
+            "index": "pypi",
+            "version": "==1.3.7"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+            ],
+            "version": "==1.25.3"
+        }
+    },
+    "develop": {}
+}

app/__init__.py

@@ -0,0 +1,28 @@
+"""Inwoner Residency en Locatie Tracker"""
+
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from dotenv import load_dotenv
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from app.models import Base, State, Region, Player
+
+
+load_dotenv()
+
+# database
+engine = create_engine(os.environ["DATABASE_URI"], client_encoding='utf8')
+Session = sessionmaker(bind=engine)
+session = Session()
+
+# scheduler
+scheduler = BackgroundScheduler()
+scheduler.start()
+
+# api
+BASE_URL = os.environ["API_URL"]
+HEADERS = {
+    'Authorization': os.environ["AUTHORIZATION"]
+}

app/__main__.py

@@ -0,0 +1,64 @@
+"""Main app"""
+
+import time
+
+from app import scheduler, session
+from app.api import get_citizens, get_residents
+from app.database import get_state_regions # , save_citizens, save_residents
+
+
+def print_players(players):
+    """Print professors"""
+    for player in players:
+        print('{:20} {:30}'.format(
+            player['id'],
+            player['name'],
+        ))
+
+def job_update_citizens(state_id):
+    """Update citizens"""
+    regions = get_state_regions(state_id)
+    for region in regions:
+        citizens = get_citizens(region.id)
+        print_players(citizens)
+        # save_citizens(citizens)
+
+def job_update_residents(state_id):
+    """Update residents"""
+    regions = get_state_regions(state_id)
+    for region in regions:
+        residents = get_residents(region.id)
+        print_players(residents)
+        # save_citizens(citizens)
+
+
+def add_update_department(state_id):
+    """Add jobs"""
+    scheduler.add_job(
+        job_update_citizens,
+        'cron',
+        args=[state_id],
+        id='citizens_{}'.format(state_id),
+        replace_existing=True,
+        minute='0'
+    )
+
+if __name__ == '__main__':
+    # jobs
+    # job_update_citizens(2788)
+    job_update_residents(2788)
+
+    # Verenigde Nederlanden
+    add_update_department(2788)
+    # Belgium
+    add_update_department(2604)
+    # De Provincien
+    add_update_department(2620)
+
+    try:
+        while True:
+            time.sleep(100)
+    except KeyboardInterrupt:
+        print('Exiting application')
+        session.close()
+        exit()

app/api.py

@@ -0,0 +1,51 @@
+"""API module"""
+
+import re
+
+import requests
+from bs4 import BeautifulSoup
+
+from app import BASE_URL, HEADERS
+
+def get_residents(region_id):
+    """Get residents from region"""
+    return download_players(region_id, 'residency')
+
+def get_citizens(region_id):
+    """Get citizens from region"""
+    return download_players(region_id, 'state')
+
+def download_players(region_id, player_type):
+    """Download the players"""
+    players = []
+    more = True
+    page = 0
+    while more:
+        response = requests.get(
+            '{}listed/{}/{}/0/{}'.format(BASE_URL, player_type, region_id, page*25),
+            headers=HEADERS
+        )
+        tmp_players, more = parse_players(response.text)
+        players = players + tmp_players
+        page += 1
+    return players
+
+def read_citizens():
+    """Read from department file"""
+    with open('citizens.html') as file:
+        citizens = parse_players(file)
+        return citizens
+
+def parse_players(html):
+    """Parse html return players"""
+    soup = BeautifulSoup(html, 'html.parser')
+    players_tree = soup.find_all(class_='list_link')
+    players = []
+    for player_tree in players_tree:
+        columns = player_tree.find_all('td')
+        players.append({
+            'id': int(re.sub(r'^.*\/', '', columns[1]['action'])),
+            'name': columns[1].string.strip(),
+        })
+    more = bool(soup.find(class_='more'))
+    return players, more

app/database.py

@@ -0,0 +1,57 @@
+"""Database module"""
+
+from app import session
+from app.models import State, Region, Player, StateRegion
+
+
+def get_state_regions(state_id):
+    """Get regions from state"""
+    state = session.query(State).get(state_id)
+    regions = state.regions.filter(StateRegion.until_date_time == None).all()
+    return regions
+
+# def get_latest_professor(state_id, department_type):
+#     """Get latest professor from database"""
+#     department = get_department(state_id, department_type)
+#     professor = department.department_stats.order_by(DepartmentStat.date_time.desc()).first()
+#     return professor
+#
+# def get_player(player_id, player_name):
+#     """Get player from database"""
+#     player = session.query(Player).get(player_id)
+#     if player is None:
+#         player = Player()
+#         player.id = player_id
+#         player.name = player_name
+#         session.add(player)
+#         session.commit()
+#     return player
+#
+# def get_department(state_id, department_type):
+#     """Get department from database"""
+#     department = session.query(Department).filter(
+#         Department.state_id == state_id
+#     ).filter(
+#         Department.department_type == department_type
+#     ).first()
+#     if department is None:
+#         department = Department()
+#         department.state_id = state_id
+#         department.department_type = department_type
+#         session.add(department)
+#         session.commit()
+#     return department
+#
+# def save_professors(state_id, department_type, professors):
+#     """Save professors to database"""
+#     department = get_department(state_id, department_type)
+#
+#     for professor in professors:
+#         player = get_player(professor['id'], professor['name'])
+#         department_stat = DepartmentStat()
+#         department_stat.department_id = department.id
+#         department_stat.date_time = professor['date_time']
+#         department_stat.points = professor['points']
+#         department_stat.player_id = player.id
+#         session.add(department_stat)
+#     session.commit()

app/models.py

@@ -0,0 +1,70 @@
+"""Database models"""
+
+from sqlalchemy import MetaData, Column, ForeignKey, Integer, String, \
+    SmallInteger, DateTime, BigInteger
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+
+class PlayerResidency(Base):
+    """Model for player residency"""
+    __tablename__ = 'player_residency'
+    player_id = Column(BigInteger, ForeignKey('player.id'), primary_key=True)
+    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True)
+    from_date_time = Column(DateTime)
+    until_date_time = Column(DateTime)
+
+
+class PlayerLocation(Base):
+    """Model for player location"""
+    __tablename__ = 'player_location'
+    player_id = Column(BigInteger, ForeignKey('player.id'), primary_key=True)
+    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True)
+    from_date_time = Column(DateTime)
+    until_date_time = Column(DateTime)
+
+
+class StateRegion(Base):
+    """Model for state region"""
+    __tablename__ = 'state_region'
+    state_id = Column(Integer, ForeignKey('state.id'), primary_key=True)
+    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True)
+    from_date_time = Column(DateTime)
+    until_date_time = Column(DateTime)
+
+
+class State(Base):
+    """Model for state"""
+    __tablename__ = 'state'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    regions = relationship(
+        'Region',
+        secondary='state_region',
+        backref=backref('states', lazy='dynamic'),
+        lazy='dynamic'
+    )
+    capital_id = Column(Integer, ForeignKey('region.id'))
+    capital = relationship(
+        'Region',
+        backref=backref('state_capital', lazy='dynamic')
+    )
+
+
+class Region(Base):
+    """Model for region"""
+    __tablename__ = 'region'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+
+class Player(Base):
+    """Model for player"""
+    __tablename__ = 'player'
+    id = Column(BigInteger, primary_key=True)
+    name = Column(String)
+    nation = Column(String)
+    residencies = relationship('Region', secondary='player_residency')
+    locations = relationship('Region', secondary='player_location')

example.env

@@ -0,0 +1,3 @@
+AUTHORIZATION=PLACEHOLDER
+DATABASE_URI='postgresql://hvs@localhost/hvs'
+API_URL='http://localhost:5000/api/request/'