From 170a2ee6ec955a5cb8ab99c6675b947b3d5fd116 Mon Sep 17 00:00:00 2001 From: tanshu Date: Sat, 25 Feb 2017 00:55:47 +0530 Subject: [PATCH] Initial Commit --- .gitignore | 6 +++ CHANGES.txt | 4 ++ MANIFEST.in | 2 + README.txt | 51 +++++++++++++++++++++ development.ini | 65 ++++++++++++++++++++++++++ install.sh | 4 ++ production.ini | 66 +++++++++++++++++++++++++++ randy.service | 15 ++++++ randy/__init__.py | 12 +++++ randy/models/__init__.py | 73 +++++++++++++++++++++++++++++ randy/models/guidtype.py | 57 +++++++++++++++++++++++ randy/models/meta.py | 16 +++++++ randy/models/picture.py | 46 +++++++++++++++++++ randy/routes.py | 3 ++ randy/scripts/__init__.py | 1 + randy/scripts/initializedb.py | 36 +++++++++++++++ randy/scripts/scan.py | 86 +++++++++++++++++++++++++++++++++++ randy/views.py | 31 +++++++++++++ setup.py | 49 ++++++++++++++++++++ 19 files changed, 623 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGES.txt create mode 100644 MANIFEST.in create mode 100644 README.txt create mode 100644 development.ini create mode 100644 install.sh create mode 100644 production.ini create mode 100644 randy.service create mode 100644 randy/__init__.py create mode 100644 randy/models/__init__.py create mode 100644 randy/models/guidtype.py create mode 100644 randy/models/meta.py create mode 100644 randy/models/picture.py create mode 100644 randy/routes.py create mode 100644 randy/scripts/__init__.py create mode 100644 randy/scripts/initializedb.py create mode 100644 randy/scripts/scan.py create mode 100644 randy/views.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1a38ccb --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__ +*.py[cod] +/.tox +/env*/ +.idea/ +*.egg-info/ \ No newline at end of file diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..14b902f --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,4 @@ +0.0 +--- + +- Initial version. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0170f70 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include *.txt *.ini *.cfg *.rst +recursive-include randy *.ico *.txt diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..ff2b783 --- /dev/null +++ b/README.txt @@ -0,0 +1,51 @@ +randy +=============================== + +Getting Started +--------------- + +- Change directory into your newly created project. + + cd randy + +- Create a Python virtual environment. + + python3 -m venv env + +- Upgrade packaging tools. + + env/bin/pip install --upgrade pip setuptools + +- Install the project in editable mode. + + env/bin/pip install -e + +- Configure the database. + + env/bin/initialize_randy_db development.ini + +- Once configured, scan the files by running the script. + + env/bin/scan development.ini + +- Run your project. + + env/bin/pserve development.ini + + +Picture Model +--------------- + +- Uri: +Format: +name_of_directory_added:path/name +eg. +Pictures:Vanya/Italy/DSC001.jpg + +- Hash +Format: +default is sha-2 +algorithm:hash + +- Weight +Default value is 4096. Every thumbs up double value, every thumbs down will halve it. diff --git a/development.ini b/development.ini new file mode 100644 index 0000000..dae3a1e --- /dev/null +++ b/development.ini @@ -0,0 +1,65 @@ +### +# app configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### + +[app:main] +use = egg:randy + +pyramid.reload_templates = true +pyramid.debug_authorization = false +pyramid.debug_notfound = false +pyramid.debug_routematch = false +pyramid.default_locale_name = en + +sqlalchemy.url = postgresql://postgres:123456@localhost:5432/randy + +directories = Pen Drive:c:\Users\tanshu\Desktop\Pen Drive\ + Pictures:c:\Users\tanshu\Pictures\ +### +# wsgi server configuration +### + +[server:main] +use = egg:waitress#main +listen = 127.0.0.1:5000 [::1]:5000 + +### +# logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, randy, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console + +[logger_randy] +level = DEBUG +handlers = +qualname = randy + +[logger_sqlalchemy] +level = INFO +handlers = +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..7aab635 --- /dev/null +++ b/install.sh @@ -0,0 +1,4 @@ +python3 -m venv env +./env/bin/pip install --upgrade pip +./env/bin/pip install -e . + diff --git a/production.ini b/production.ini new file mode 100644 index 0000000..534d240 --- /dev/null +++ b/production.ini @@ -0,0 +1,66 @@ +### +# app configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### + +[app:main] +use = egg:randy + +pyramid.reload_templates = false +pyramid.debug_authorization = false +pyramid.debug_notfound = false +pyramid.debug_routematch = false +pyramid.default_locale_name = en + +sqlalchemy.url = postgresql://postgres:123456@localhost:5432/randy + +directories = Pictures:/home/user/Pictures/ + Others:c:\Users\user\Pictures\ + +### +# wsgi server configuration +### + +[server:main] +use = egg:waitress#main +listen = *:5000 + +### +# logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, randy, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_randy] +level = WARN +handlers = +qualname = randy + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/randy.service b/randy.service new file mode 100644 index 0000000..27d6db4 --- /dev/null +++ b/randy.service @@ -0,0 +1,15 @@ +[Unit] +Description=Tanshu's Random Image Server +After=multi-user.target + +[Service] +Type=simple +Restart=always +ExecStart=/home/tanshu/Programming/randy/env/bin/pserve production.ini +User=tanshu +WorkingDirectory=/home/tanshu/Programming/randy +ExecStop=/usr/bin/pkill -f /home/tanshu/Programming/randy/env/bin/pserve + +[Install] +WantedBy=multi-user.target + diff --git a/randy/__init__.py b/randy/__init__.py new file mode 100644 index 0000000..9f4c226 --- /dev/null +++ b/randy/__init__.py @@ -0,0 +1,12 @@ +from pyramid.config import Configurator + + +def main(global_config, **settings): + """ This function returns a Pyramid WSGI application. + """ + config = Configurator(settings=settings) + config.include('.models') + config.include('.routes') + config.scan() + return config.make_wsgi_app() + diff --git a/randy/models/__init__.py b/randy/models/__init__.py new file mode 100644 index 0000000..a24733a --- /dev/null +++ b/randy/models/__init__.py @@ -0,0 +1,73 @@ +from sqlalchemy import engine_from_config +from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import configure_mappers +import zope.sqlalchemy + +# import or define all models here to ensure they are attached to the +# Base.metadata prior to any initialization routines +from .picture import Picture # flake8: noqa + +# run configure_mappers after defining all of the models to ensure +# all relationships can be setup +configure_mappers() + + +def get_engine(settings, prefix='sqlalchemy.'): + return engine_from_config(settings, prefix) + + +def get_session_factory(engine): + factory = sessionmaker() + factory.configure(bind=engine) + return factory + + +def get_tm_session(session_factory, transaction_manager): + """ + Get a ``sqlalchemy.orm.Session`` instance backed by a transaction. + + This function will hook the session to the transaction manager which + will take care of committing any changes. + + - When using pyramid_tm it will automatically be committed or aborted + depending on whether an exception is raised. + + - When using scripts you should wrap the session in a manager yourself. + For example:: + + import transaction + + engine = get_engine(settings) + session_factory = get_session_factory(engine) + with transaction.manager: + dbsession = get_tm_session(session_factory, transaction.manager) + + """ + dbsession = session_factory() + zope.sqlalchemy.register( + dbsession, transaction_manager=transaction_manager) + return dbsession + + +def includeme(config): + """ + Initialize the model for a Pyramid app. + + Activate this setup using ``config.include('randy.models')``. + + """ + settings = config.get_settings() + + # use pyramid_tm to hook the transaction lifecycle to the request + config.include('pyramid_tm') + + session_factory = get_session_factory(get_engine(settings)) + config.registry['dbsession_factory'] = session_factory + + # make request.dbsession available for use in Pyramid + config.add_request_method( + # r.tm is the transaction manager used by pyramid_tm + lambda r: get_tm_session(session_factory, r.tm), + 'dbsession', + reify=True + ) diff --git a/randy/models/guidtype.py b/randy/models/guidtype.py new file mode 100644 index 0000000..73aebbc --- /dev/null +++ b/randy/models/guidtype.py @@ -0,0 +1,57 @@ +import uuid +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.dialects.sqlite import BLOB +from sqlalchemy.types import TypeDecorator, CHAR, Binary + + +class GUID(TypeDecorator): + """Platform-independent GUID type. + + Uses Postgresql's UUID type, otherwise uses + CHAR(32), storing as stringified hex values. + + """ + impl = Binary + + # if dialect.value == 'postgresql': + # impl = CHAR + # elif dialect.value == 'mysql': + # impl = MSBinary + # elif dialect.valie == 'sqlite': + # impl = Binary + # else: + # impl = Binary + + def load_dialect_impl(self, dialect): + if dialect.name == 'postgresql': + return dialect.type_descriptor(UUID()) + elif dialect.name == 'sqlite': + return dialect.type_descriptor(BLOB()) + else: + return dialect.type_descriptor(CHAR(32)) + + def process_bind_param(self, value, dialect): + if value is None: + return None + elif dialect.name == 'postgresql': + return str(value) + elif not isinstance(value, uuid.UUID): + raise ValueError('value %s is not a valid uuid.UUID' % value) + else: + return value.bytes + # if not isinstance(value, uuid.UUID): + # return "%.32x" % uuid.UUID(value) + # else: + # # hexstring + # return "%.32x" % value + + def process_result_value(self, value, dialect=None): + if value is None: + return None + elif isinstance(value, bytes): + return uuid.UUID(bytes=value) + else: + return uuid.UUID(value) + + def is_mutable(self): + return False diff --git a/randy/models/meta.py b/randy/models/meta.py new file mode 100644 index 0000000..0682247 --- /dev/null +++ b/randy/models/meta.py @@ -0,0 +1,16 @@ +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import MetaData + +# Recommended naming convention used by Alembic, as various different database +# providers will autogenerate vastly different names making migrations more +# difficult. See: http://alembic.zzzcomputing.com/en/latest/naming.html +NAMING_CONVENTION = { + "ix": 'ix_%(column_0_label)s', + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s" +} + +metadata = MetaData(naming_convention=NAMING_CONVENTION) +Base = declarative_base(metadata=metadata) diff --git a/randy/models/picture.py b/randy/models/picture.py new file mode 100644 index 0000000..3e5f769 --- /dev/null +++ b/randy/models/picture.py @@ -0,0 +1,46 @@ +import uuid + +from sqlalchemy import ( + Column, + Integer, + Unicode, + DateTime, + SmallInteger, + UniqueConstraint +) +from sqlalchemy.dialects.postgresql import JSON + +from randy.models.meta import Base +from randy.models.guidtype import GUID + + +class Picture(Base): + __tablename__ = 'pictures' + __table_args__ = (UniqueConstraint('modified_time', 'uri', 'hash', 'size'), ) + + id = Column('id', GUID(), primary_key=True, default=uuid.uuid4) + uri = Column('uri', Unicode(255), unique=True) + weight = Column('weight', SmallInteger) + file_hash = Column('hash', Unicode(255)) + exif = Column('exif', JSON) + cloud_vision = Column('cloud_vision', JSON) + size = Column('size', Integer) + modified_time = Column('modified_time', DateTime) + + def __init__(self, + uri=None, + weight=4096, + file_hash=None, + exif=None, + cloud_vision=None, + size=None, + modified_date=None, + picture_id=None): + self.uri = uri + self.weight = weight + self.file_hash = file_hash + self.exif = exif + self.cloud_vision = cloud_vision + self.size = size + self.modified_date = modified_date + self.id = picture_id diff --git a/randy/routes.py b/randy/routes.py new file mode 100644 index 0000000..b1aa49b --- /dev/null +++ b/randy/routes.py @@ -0,0 +1,3 @@ +def includeme(config): + config.add_route('random', '/random') + config.add_route('id', '/{id}') diff --git a/randy/scripts/__init__.py b/randy/scripts/__init__.py new file mode 100644 index 0000000..5bb534f --- /dev/null +++ b/randy/scripts/__init__.py @@ -0,0 +1 @@ +# package diff --git a/randy/scripts/initializedb.py b/randy/scripts/initializedb.py new file mode 100644 index 0000000..28f6187 --- /dev/null +++ b/randy/scripts/initializedb.py @@ -0,0 +1,36 @@ +import os +import sys +import transaction + +from pyramid.paster import ( + get_appsettings, + setup_logging, + ) + +from pyramid.scripts.common import parse_vars + +from ..models.meta import Base +from ..models import ( + get_engine, + get_session_factory, + get_tm_session, + ) + + +def usage(argv): + cmd = os.path.basename(argv[0]) + print('usage: %s [var=value]\n' + '(example: "%s development.ini")' % (cmd, cmd)) + sys.exit(1) + + +def main(argv=sys.argv): + if len(argv) < 2: + usage(argv) + config_uri = argv[1] + options = parse_vars(argv[2:]) + setup_logging(config_uri) + settings = get_appsettings(config_uri, options=options) + + engine = get_engine(settings) + Base.metadata.create_all(engine) diff --git a/randy/scripts/scan.py b/randy/scripts/scan.py new file mode 100644 index 0000000..74998d6 --- /dev/null +++ b/randy/scripts/scan.py @@ -0,0 +1,86 @@ +import glob +import hashlib +import os +import sys +import time + +import transaction +from pyramid.paster import ( + get_appsettings, + setup_logging, +) +from pyramid.scripts.common import parse_vars +from ..models import ( + get_engine, + get_session_factory, + get_tm_session, + Picture +) + + +def usage(argv): + cmd = os.path.basename(argv[0]) + print('usage: %s [var=value]\n' + '(example: "%s development.ini")' % (cmd, cmd)) + sys.exit(1) + + +def main(argv=sys.argv): + if len(argv) < 2: + usage(argv) + config_uri = argv[1] + options = parse_vars(argv[2:]) + setup_logging(config_uri) + settings = get_appsettings(config_uri, options=options) + + engine = get_engine(settings) + session_factory = get_session_factory(engine) + with transaction.manager: + dbsession = get_tm_session(session_factory, transaction.manager) + scan(settings['directories'].splitlines(), dbsession) + + +def scan(directories, dbsession): + for item in directories: + name, path = item.split(':', 1) + getFiles(name, path, dbsession) + + +def getFiles(name, path, dbsession): + files = glob.glob(path + '**/*.jpg', recursive=True) + for file in files: + uri = name + ':' + file[len(path):] + + size = os.path.getsize(file) + modified_time = time.gmtime(os.path.getmtime(file)) + old = dbsession.query(Picture).filter(Picture.uri == uri).first() + # if old is not None and old.size == size and old.modified_time == modified_time: + if old is not None: + print('Old file with ' + old.file_hash + ' exists!') + continue + file_hash = getHash(file) + # if old is not None: + # old.size = size + # old.modified_time = modified_time + # print('updated old', old.modified_time, 'with', modified_time) + # old.file_hash = file_hash + # print('Updated file with ' + file_hash) + # else: + picture = Picture(uri, 4096, file_hash, {}, {}, size, modified_time) + dbsession.add(picture) + print('Added file with ' + file_hash) + + +def getHash(file): + # BUF_SIZE is totally arbitrary, change for your app! + BUF_SIZE = 1 * 1024 * 1024 # lets read stuff in 64kb chunks! + sha256 = hashlib.sha256() + + with open(file, 'rb') as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha256.update(data) + + return "SHA256:{0}".format(sha256.hexdigest()) diff --git a/randy/views.py b/randy/views.py new file mode 100644 index 0000000..727a45c --- /dev/null +++ b/randy/views.py @@ -0,0 +1,31 @@ +import random + +from pyramid.response import Response, FileResponse +from pyramid.view import view_config +from sqlalchemy.exc import DBAPIError +from .models import Picture + + +@view_config(route_name='random') +def random_view(request): + try: + chosen = weighted_choice(request.dbsession.query(Picture.id, Picture.weight).all()) + item = request.dbsession.query(Picture).filter(Picture.id == chosen).first() + except DBAPIError: + return Response("", content_type='text/plain', status=500) + directories = request.registry.settings['directories'].splitlines() + repo, path = item.uri.split(':', 1) + location = [i for i in directories if i.startswith(repo)][0] + file = location[len(repo) + 1:] + path + return FileResponse(file, request=request) + + +def weighted_choice(choices): + total = sum(w for c, w in choices) + r = random.uniform(0, total) + upto = 0 + for c, w in choices: + upto += w + if upto >= r: + return c + assert False, "Shouldn't get here" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..89449fd --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +import os + +from setuptools import setup, find_packages + +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, 'README.txt')) as f: + README = f.read() +with open(os.path.join(here, 'CHANGES.txt')) as f: + CHANGES = f.read() + +requires = [ + 'pyramid', + 'pyramid_tm', + 'SQLAlchemy', + 'psycopg2', + 'transaction', + 'zope.sqlalchemy', + 'waitress', +] + +setup( + name='randy', + version='0.0', + description='randy', + long_description=README + '\n\n' + CHANGES, + classifiers=[ + 'Programming Language :: Python', + 'Framework :: Pyramid', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: WSGI :: Application', + ], + author='', + author_email='', + url='', + keywords='web pyramid pylons', + packages=find_packages(), + include_package_data=True, + zip_safe=False, + install_requires=requires, + entry_points={ + 'paste.app_factory': [ + 'main = randy:main', + ], + 'console_scripts': [ + 'initialize_db = randy.scripts.initializedb:main', + 'scan = randy.scripts.scan:main', + ], + }, +)