Inital commit

.gitignore

@@ -0,0 +1,3 @@
+.venv/
+.env
+__pycache__

Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = "*"
+beautifulsoup4 = "*"
+sqlalchemy = "*"
+python-dotenv = "*"
+psycopg2-binary = "*"
+apscheduler = "*"
+
+[requires]
+python_version = "3.7"

Pipfile.lock

@@ -0,0 +1,151 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "40681423ee7bb7705184a16526334130adc76c1b726852ea6797a4552c90cdac"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "apscheduler": {
+            "hashes": [
+                "sha256:529afb7909e08416132891188cbfea5351eb35e4a684b67e983d819e8d01a6b0",
+                "sha256:cde18f6dbffa1b75aff67fd7fe423a3020cb0363f6c67bd45f24306d90898231"
+            ],
+            "index": "pypi",
+            "version": "==3.6.1"
+        },
+        "beautifulsoup4": {
+            "hashes": [
+                "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
+                "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
+                "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
+            ],
+            "index": "pypi",
+            "version": "==4.8.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
+                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+            ],
+            "version": "==2019.6.16"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:080c72714784989474f97be9ab0ddf7b2ad2984527e77f2909fcd04d4df53809",
+                "sha256:110457be80b63ff4915febb06faa7be002b93a76e5ba19bf3f27636a2ef58598",
+                "sha256:171352a03b22fc099f15103959b52ee77d9a27e028895d7e5fde127aa8e3bac5",
+                "sha256:19d013e7b0817087517a4b3cab39c084d78898369e5c46258aab7be4f233d6a1",
+                "sha256:249b6b21ae4eb0f7b8423b330aa80fab5f821b9ffc3f7561a5e2fd6bb142cf5d",
+                "sha256:2ac0731d2d84b05c7bb39e85b7e123c3a0acd4cda631d8d542802c88deb9e87e",
+                "sha256:2b6d561193f0dc3f50acfb22dd52ea8c8dfbc64bcafe3938b5f209cc17cb6f00",
+                "sha256:2bd23e242e954214944481124755cbefe7c2cf563b1a54cd8d196d502f2578bf",
+                "sha256:3e1239242ca60b3725e65ab2f13765fc199b03af9eaf1b5572f0e97bdcee5b43",
+                "sha256:3eb70bb697abbe86b1d2b1316370c02ba320bfd1e9e35cf3b9566a855ea8e4e5",
+                "sha256:51a2fc7e94b98bd1bb5d4570936f24fc2b0541b63eccadf8fdea266db8ad2f70",
+                "sha256:52f1bdafdc764b7447e393ed39bb263eccb12bfda25a4ac06d82e3a9056251f6",
+                "sha256:5b3581319a3951f1e866f4f6c5e42023db0fae0284273b82e97dfd32c51985cd",
+                "sha256:63c1b66e3b2a3a336288e4bcec499e0dc310cd1dceaed1c46fa7419764c68877",
+                "sha256:8123a99f24ecee469e5c1339427bcdb2a33920a18bb5c0d58b7c13f3b0298ba3",
+                "sha256:85e699fcabe7f817c0f0a412d4e7c6627e00c412b418da7666ff353f38e30f67",
+                "sha256:8dbff4557bbef963697583366400822387cccf794ccb001f1f2307ed21854c68",
+                "sha256:908d21d08d6b81f1b7e056bbf40b2f77f8c499ab29e64ec5113052819ef1c89b",
+                "sha256:af39d0237b17d0a5a5f638e9dffb34013ce2b1d41441fd30283e42b22d16858a",
+                "sha256:af51bb9f055a3f4af0187149a8f60c9d516cf7d5565b3dac53358796a8fb2a5b",
+                "sha256:b2ecac57eb49e461e86c092761e6b8e1fd9654dbaaddf71a076dcc869f7014e2",
+                "sha256:cd37cc170678a4609becb26b53a2bc1edea65177be70c48dd7b39a1149cabd6e",
+                "sha256:d17e3054b17e1a6cb8c1140f76310f6ede811e75b7a9d461922d2c72973f583e",
+                "sha256:d305313c5a9695f40c46294d4315ed3a07c7d2b55e48a9010dad7db7a66c8b7f",
+                "sha256:dd0ef0eb1f7dd18a3f4187226e226a7284bda6af5671937a221766e6ef1ee88f",
+                "sha256:e1adff53b56db9905db48a972fb89370ad5736e0450b96f91bcf99cadd96cfd7",
+                "sha256:f0d43828003c82dbc9269de87aa449e9896077a71954fbbb10a614c017e65737",
+                "sha256:f78e8b487de4d92640105c1389e5b90be3496b1d75c90a666edd8737cc2dbab7"
+            ],
+            "index": "pypi",
+            "version": "==2.8.3"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:debd928b49dbc2bf68040566f55cdb3252458036464806f4094487244e2a4093",
+                "sha256:f157d71d5fec9d4bd5f51c82746b6344dffa680ee85217c123f4a0c8117c4544"
+            ],
+            "index": "pypi",
+            "version": "==0.10.3"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "index": "pypi",
+            "version": "==2.22.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+            ],
+            "version": "==1.12.0"
+        },
+        "soupsieve": {
+            "hashes": [
+                "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
+                "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
+            ],
+            "version": "==1.9.3"
+        },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
+            ],
+            "index": "pypi",
+            "version": "==1.3.7"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+            ],
+            "version": "==1.25.3"
+        }
+    },
+    "develop": {}
+}

app/__init__.py

@@ -0,0 +1,27 @@
+"""Init"""
+
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from dotenv import load_dotenv
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from app.models import Base, Player, State, Department, DepartmentStat
+
+
+load_dotenv()
+
+# database
+engine = create_engine(os.environ["DATABASE_URI"])
+Session = sessionmaker(bind=engine)
+
+# scheduler
+scheduler = BackgroundScheduler()
+scheduler.start()
+
+# api
+BASE_URL = os.environ["API_URL"]
+HEADERS = {
+    'Authorization': os.environ["AUTHORIZATION"]
+}

app/__main__.py

@@ -0,0 +1,33 @@
+"""Main app"""
+
+from datetime import datetime, timedelta
+import random
+import time
+
+from app import scheduler
+from app.api import get_professors
+from app.database import get_latest_professor, save_professors
+
+
+def job_update_department(state_id, department_type):
+    """Update department professors"""
+    # last_professor = get_last_professor(state_id, department_type)
+    professors = get_professors(state_id, department_type, None)
+    print(professors)
+    # save_professors(state_id, department_type, professors)
+
+
+if __name__ == '__main__':
+    # jobs
+    job_update_department(2788, 6)
+    scheduler.add_job(
+        job_update_department,
+        'cron',
+        args=[2788, 6],
+        id='vn_update_department',
+        replace_existing=True,
+        hour='20'
+    )
+
+    while True:
+        time.sleep(100)

app/api.py

@@ -0,0 +1,61 @@
+"""API module"""
+
+import re 
+
+import requests
+from bs4 import BeautifulSoup
+
+from app import BASE_URL, HEADERS
+
+def get_professors(state_id, department_type, start_date):
+    """Download list of professors"""
+    professors = []
+    not_reached_date = True
+    page = 0
+    while not_reached_date:
+        # tmp_professors = download_department(state_id, department_type, page)
+        tmp_professors = read_department()
+        for professor in tmp_professors:
+            if start_date is not None: # and start_date >=:
+                not_reached_date = False
+                break
+            professors.append(professor)
+
+        page += 1
+        break
+
+    return professors
+
+def download_department(state_id, department_type, page):
+    """Download the department"""
+    response = requests.get(
+        '{}listed/professors/{}/{}/{}'.format(BASE_URL, department_type, state_id, page*25),
+        headers=HEADERS
+    )
+    return parse_department(response.text)
+
+def read_department():
+    """Read from department file"""
+    with open('department.html') as file:
+        return parse_department(file)
+
+def parse_department(html):
+    """Parse html return professors"""
+    soup = BeautifulSoup(html, 'html.parser')
+    professors_tree = soup.find_all(class_='list_link')
+    print(professors_tree)
+    professors = []
+    for professor_tree in professors_tree:
+        print(professor_tree)
+        columns = professor_tree.find_all('td')
+        professors.append(
+            {
+                'id': int(professor_tree['user']),
+                'name': re.sub(r'\s\(.*$', '', columns[1].string),
+                'points': int(re.sub(r'^.*\(\+|\)$', '', columns[1].string)),
+                'date': columns[3].string,
+            }
+        )
+    print(professors)
+    exit()
+    return professors

app/database.py

@@ -0,0 +1,26 @@
+"""Database module"""
+
+from app import Session
+from app.models import Player, State, Department, DepartmentStat
+
+
+def get_latest_professor(state_id, department_type):
+    """Get latest professor from database"""
+    return None
+
+def save_professors(state_id, department_type, professors):
+    """Save professors to database"""
+    session = Session()
+    resource_track.state_id = state_id
+    resource_track.resource_id = resource_id
+    session.add(resource_track)
+    session.commit()
+
+    for region_id, region in regions.items():
+        resource_stat = ResourceStat()
+        resource_stat.region_id = region_id
+        resource_stat.explored = region['explored']
+        resource_stat.deep_exploration = region['deep_exploration']
+        resource_stat.limit_left = region['limit_left']
+        session.add(resource_stat)
+    session.commit()

app/models.py

@@ -0,0 +1,64 @@
+"""Database models"""
+
+from sqlalchemy import MetaData, Column, ForeignKey, Integer, String, SmallInteger, DateTime
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+
+
+meta = MetaData(naming_convention={
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s"
+})
+Base = declarative_base(metadata=meta)
+
+
+class Player(Base):
+    """Model for player"""
+    __tablename__ = 'player'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    nation = Column(String)
+
+
+class State(Base):
+    """Model for state"""
+    __tablename__ = 'state'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+
+
+class Department(Base):
+    """Model for department"""
+    __tablename__ = 'department'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    department_type = Column(Integer)
+
+    state_id = Column(Integer, ForeignKey('state.id'))
+    state = relationship(
+        'State',
+        backref=backref('elections', lazy='dynamic')
+    )
+
+
+class DepartmentStat(Base):
+    """Model for departent stat"""
+    __tablename__ = 'department_stat'
+    id = Column(Integer, primary_key=True)
+    date_time = Column(DateTime)
+    points = Column(SmallInteger)
+
+    player_id = Column(Integer, ForeignKey('player.id'))
+    player = relationship(
+        'Player',
+        backref=backref('department_stats', lazy='dynamic')
+    )
+
+    department_id = Column(Integer, ForeignKey('department.id'))
+    department = relationship(
+        'Department',
+        backref=backref('stats', lazy='dynamic')
+    )

example.env

@@ -0,0 +1,3 @@
+AUTHORIZATION=PLACEHOLDER
+DATABASE_URI='postgresql://hvs@localhost/hvs'
+API_URL='http://localhost:5000/api/request/'