diff --git a/.gitignore b/.gitignore index 8e937b9e..782d3d65 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ db.sqlite3 db.sqlite3-journal .idea/* venv +*.code-workspace diff --git a/Pipfile b/Pipfile index ceb97ab5..f740fc81 100644 --- a/Pipfile +++ b/Pipfile @@ -8,7 +8,7 @@ verify_ssl = true [packages] django-picklefield = "*" lxml = "*" -python-telegram-bot = ">=10" +python-telegram-bot = ">=13.5" Django = ">=1.10" Pillow = ">=2.9.0" APScheduler = ">=3.3.0" diff --git a/opds_catalog/dl.py b/opds_catalog/dl.py index 132b0a91..0bca332a 100644 --- a/opds_catalog/dl.py +++ b/opds_catalog/dl.py @@ -19,6 +19,8 @@ from constance import config from PIL import Image +from opds_catalog.middleware import BasicAuthMiddleware + def getFileName(book): if config.SOPDS_TITLE_AS_FILENAME: transname = utils.translit(book.title + '.' + book.format) @@ -143,8 +145,14 @@ def Download(request, book_id, zip_flag): """ Загрузка файла книги """ book = Book.objects.get(id=book_id) - if config.SOPDS_AUTH and request.user.is_authenticated: - bookshelf.objects.get_or_create(user=request.user, book=book) + if config.SOPDS_AUTH: + if not request.user.is_authenticated: + bau = BasicAuthMiddleware() + request = bau.process_request(request) + if not hasattr(request, 'user'): + return request + if request.user.is_authenticated: + bookshelf.objects.get_or_create(user=request.user, book=book) full_path=os.path.join(config.SOPDS_ROOT_LIB,book.path) diff --git a/opds_catalog/feeds.py b/opds_catalog/feeds.py index dd89e294..0c5f3946 100644 --- a/opds_catalog/feeds.py +++ b/opds_catalog/feeds.py @@ -30,7 +30,7 @@ def __call__(self,request,*args,**kwargs): bau = BasicAuthMiddleware() result=bau.process_request(self.request) - if result!=None: + if (result != None) and (not hasattr(result, 'user')): return result return super().__call__(request,*args,**kwargs) diff --git a/opds_catalog/management/commands/sopds_telebot.py b/opds_catalog/management/commands/sopds_telebot.py index cdfd1bb5..ab5eb9b4 100644 --- a/opds_catalog/management/commands/sopds_telebot.py +++ b/opds_catalog/management/commands/sopds_telebot.py @@ -4,40 +4,47 @@ import logging import re +import json + +from collections import OrderedDict +from datetime import datetime, timedelta + from django.core.management.base import BaseCommand from django.conf import settings as main_settings from django.utils.html import strip_tags -from django.db.models import Q +from django.db.models import Q, Count, Max from django.db import transaction, connection, connections from django.contrib.auth.models import User from django.utils.translation import ugettext as _ from django.utils import translation +from django.contrib.postgres.aggregates import StringAgg + from opds_catalog.models import Book from opds_catalog import settings, dl from opds_catalog.opds_paginator import Paginator as OPDS_Paginator from sopds_web_backend.settings import HALF_PAGES_LINKS from constance import config -from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, RegexHandler, CallbackQueryHandler -from telegram import InlineKeyboardButton, InlineKeyboardMarkup -from telegram.error import InvalidToken +from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackQueryHandler, CallbackContext +from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update +from telegram.error import InvalidToken, BadRequest query_delimiter = "####" def cmdtrans(func): - def wrapper(self, bot, update): + def wrapper(self, update: Update, context: CallbackContext): translation.activate(config.SOPDS_LANGUAGE) - result = func(self, bot, update) + result = func(self, update, context) translation.deactivate() return result return wrapper -def CheckAuthDecorator(func): - def wrapper(self, bot, update): +def check_auth_decorator(func): + def wrapper(self, update: Update, context: CallbackContext): if not config.SOPDS_TELEBOT_AUTH: - return func(self, bot, update) + return func(self, update, context) if connection.connection and not connection.is_usable(): del(connections._connections.default) @@ -47,18 +54,24 @@ def wrapper(self, bot, update): users = User.objects.filter(username__iexact=username) if users and users[0].is_active: - return func(self, bot, update) + return func(self, update, context) - bot.sendMessage(chat_id=query.chat_id, + context.bot.sendMessage(chat_id=query.chat_id, text=_("Hello %s!\nUnfortunately you do not have access to information. Please contact the bot administrator.") % username) self.logger.info(_("Denied access for user: %s") % username) - return + return None return wrapper class Command(BaseCommand): help = 'SimpleOPDS Telegram Bot engine.' + + # The above code is creating a variable named "query_cache". + query_cache = OrderedDict() + query_cache_max_size = 10 + query_cache_max_age = timedelta(days = 2) + can_import_settings = True leave_locale_alone = True @@ -66,10 +79,11 @@ def add_arguments(self, parser): parser.add_argument('command', help='Use [ start | stop | restart ]') parser.add_argument('--verbose',action='store_true', dest='verbose', default=False, help='Set verbosity level for SimpleOPDS telebot.') parser.add_argument('--daemon',action='store_true', dest='daemonize', default=False, help='Daemonize server') - + return None + def handle(self, *args, **options): self.pidfile = os.path.join(main_settings.BASE_DIR, config.SOPDS_TELEBOT_PID) - action = options['command'] + action = options['command'] self.logger = logging.getLogger('') self.logger.setLevel(logging.DEBUG) formatter=logging.Formatter('%(asctime)s %(levelname)-8s %(message)s') @@ -87,12 +101,12 @@ def handle(self, *args, **options): ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) self.logger.addHandler(ch) - + if (options["daemonize"] and (action in ["start"])): if sys.platform == "win32": self.stdout.write("On Windows platform Daemonize not working.") - else: - daemonize() + else: + daemonize() if action == "start": self.start() @@ -102,91 +116,71 @@ def handle(self, *args, **options): elif action == "restart": pid = open(self.pidfile, "r").read() self.restart(pid) + return None @cmdtrans - @CheckAuthDecorator - def startCommand(self, bot, update): - bot.sendMessage(chat_id=update.message.chat_id, text=_("%(subtitle)s\nHello %(username)s! To search for a book, enter part of her title or author:")% + @check_auth_decorator + def startCommand(self, update: Update, context: CallbackContext): + context.bot.sendMessage(chat_id=update.message.chat_id, text=_("%(subtitle)s\nHello %(username)s! To search for a book, enter part of her title or author:")% {'subtitle':settings.SUBTITLE,'username':update.message.from_user.username}) self.logger.info("Start talking with user: %s"%update.message.from_user) - return + return None - def BookFilter(self, query): + def bookFilter(self, query): if connection.connection and not connection.is_usable(): del(connections._connections.default) - + if query in self.query_cache: + self.query_cache.move_to_end(query) + timestamp, books = self.query_cache[query] + if datetime.now() - timestamp <= self.query_cache_max_age: + return books + else: + self.logger.info("Query '%s' is too old in query cache."%query) q_objects = Q() q_objects.add(Q(search_title__contains=query.upper()), Q.OR) q_objects.add( Q(authors__search_full_name__contains=query.upper()), Q.OR) - books = Book.objects.filter(q_objects).order_by('search_title', '-docdate').distinct() + books = Book.objects.filter(q_objects).annotate(authors_set=StringAgg("authors__full_name", delimiter=", ")) + if config.SOPDS_DOUBLES_HIDE: + books = books.values("title", "search_title", "authors_set").annotate(doubles=Count("filename"), id=Max("id")).order_by("search_title").distinct() + else: + books = books.values("title", "search_title", "authors_set", "id", "docdate").order_by('search_title', '-docdate').distinct() + self.query_cache[query] = datetime.now(), books + #self.logger.info("Query '%s' is added to query cache."%query) + if len(self.query_cache) > self.query_cache_max_size: + query_old, _books_old = self.query_cache.popitem(0) + self.logger.info("Query cache is overloaded. Query '%s' is removed from query cache."%query_old) return books - def BookPager(self, books, page_num, query): - books_count = books.count() + def bookPager(self, books, page_num, query): + # as I can understand, len de-facto reads all items in memory or QuerySet cache + books_count = len(books) op = OPDS_Paginator(books_count, 0, page_num, config.SOPDS_TELEBOT_MAXITEMS, HALF_PAGES_LINKS) - items = [] - - prev_title = '' - prev_authors_set = set() + summary_doubles = config.SOPDS_DOUBLES_HIDE - # Начаинам анализ с последнего элемента на предидущей странице, чторбы он "вытянул" с этой страницы - # свои дубликаты если они есть - summary_DOUBLES_HIDE = config.SOPDS_DOUBLES_HIDE - start = op.d1_first_pos if ((op.d1_first_pos == 0) or (not summary_DOUBLES_HIDE)) else op.d1_first_pos - 1 + start = op.d1_first_pos if (op.d1_first_pos == 0) else op.d1_first_pos - 1 finish = op.d1_last_pos - for row in books[start:finish + 1]: - p = {'doubles': 0, 'lang_code': row.lang_code, 'filename': row.filename, 'path': row.path, \ - 'registerdate': row.registerdate, 'id': row.id, 'annotation': strip_tags(row.annotation), \ - 'docdate': row.docdate, 'format': row.format, 'title': row.title, 'filesize': row.filesize // 1000, \ - 'authors': row.authors.values(), 'genres': row.genres.values(), 'series': row.series.values(), - 'ser_no': row.bseries_set.values('ser_no') - } - if summary_DOUBLES_HIDE: - title = p['title'] - authors_set = {a['id'] for a in p['authors']} - if title.upper() == prev_title.upper() and authors_set == prev_authors_set: - items[-1]['doubles'] += 1 - else: - items.append(p) - prev_title = title - prev_authors_set = authors_set - else: - items.append(p) - - # "вытягиваем" дубликаты книг со следующей страницы и удаляем первый элемент который с предыдущей страницы и "вытягивал" дубликаты с текущей - if summary_DOUBLES_HIDE: - double_flag = True - while ((finish + 1) < books_count) and double_flag: - finish += 1 - if books[finish].title.upper() == prev_title.upper() and {a['id'] for a in books[finish].authors.values()} == prev_authors_set: - items[-1]['doubles'] += 1 - else: - double_flag = False - - if op.d1_first_pos != 0: - items.pop(0) - response = '' - for b in items: - authors = ', '.join([a['full_name'] for a in b['authors']]) - doubles = _("(doubles:%s) ")%b['doubles'] if b['doubles'] else '' - response+='%(title)s\n%(author)s\n%(dbl)s/download%(link)s\n\n'%{'title':b['title'], 'author':authors,'link':b['id'], 'dbl':doubles} - - buttons = [InlineKeyboardButton('1 <<', callback_data='%s%s%s'%(query,query_delimiter,1)), - InlineKeyboardButton('%s <'%op.previous_page_number , callback_data='%s%s%s'%(query,query_delimiter,op.previous_page_number)), - InlineKeyboardButton('[ %s ]'%op.number , callback_data='%s%s%s'%(query,query_delimiter,'current')), - InlineKeyboardButton('> %s'%op.next_page_number , callback_data='%s%s%s'%(query,query_delimiter,op.next_page_number)), - InlineKeyboardButton('>> %s'%op.num_pages, callback_data='%s%s%s'%(query,query_delimiter,op.num_pages))] - - markup = InlineKeyboardMarkup([buttons]) if op.num_pages>1 else None - - return {'message':response, 'buttons':markup} + for b in books[start:finish + 1]: + doubles = _("(doubles:%s) ")%b['doubles'] if summary_doubles and b['doubles'] else '' + response+='%(title)s\n%(author)s\n%(dbl)s/download%(link)s\n\n'%{'title':b['title'], 'author':b['authors_set'],'link':b['id'], 'dbl':doubles} + + #fix for rare empty response + if response: + buttons = [InlineKeyboardButton('1 <<', callback_data='%s%s%s'%(query,query_delimiter,1)), + InlineKeyboardButton('%s <'%op.previous_page_number , callback_data='%s%s%s'%(query,query_delimiter,op.previous_page_number)), + InlineKeyboardButton('[ %s ]'%op.number , callback_data='%s%s%s'%(query,query_delimiter,'current')), + InlineKeyboardButton('> %s'%op.next_page_number , callback_data='%s%s%s'%(query,query_delimiter,op.next_page_number)), + InlineKeyboardButton('>> %s'%op.num_pages, callback_data='%s%s%s'%(query,query_delimiter,op.num_pages))] + markup = InlineKeyboardMarkup([buttons]) if op.num_pages>1 else None + return {'message':response, 'buttons':markup} + else: + return self.bookPager(books, page_num - 1, query) @cmdtrans - @CheckAuthDecorator - def getBooks(self, bot, update): + @check_auth_decorator + def getBooks(self, update: Update, context: CallbackContext): query=update.message.text self.logger.info("Got message from user %s: %s" % (update.message.from_user.username, query)) @@ -195,31 +189,34 @@ def getBooks(self, bot, update): else: response = _("I'm searching for the book: %s") % (query) - bot.send_message(chat_id=update.message.chat_id, text=response) + context.bot.send_message(chat_id=update.message.chat_id, text=response) self.logger.info("Send message to user %s: %s" % (update.message.from_user.username,response)) if len(query) < 3: - return + return None - books = self.BookFilter(query) - books_count = books.count() + books = self.bookFilter(query) + #books_count = books.count() + # as I can understand, len de-facto reads all items in memory or QuerySet cache + books_count = len(books) if books_count == 0: response = _("No results were found for your query, please try again.") - bot.send_message(chat_id=update.message.chat_id, text=response) + context.bot.send_message(chat_id=update.message.chat_id, text=response) self.logger.info("Send message to user %s: %s" % (update.message.from_user.username,response)) return response = _("Found %s books.\nI create list, after a few seconds, select the file to download:") % books_count - bot.send_message(chat_id=update.message.chat_id, text=response) + context.bot.send_message(chat_id=update.message.chat_id, text=response) self.logger.info("Send message to user %s: %s" % (update.message.from_user.username, response)) - response = self.BookPager(books, 1, query) - bot.send_message(chat_id=update.message.chat_id, text=response['message'], parse_mode='HTML', reply_markup=response['buttons']) + response = self.bookPager(books, 1, query) + context.bot.send_message(chat_id=update.message.chat_id, text=response['message'], parse_mode='HTML', reply_markup=response['buttons']) + return None @cmdtrans - @CheckAuthDecorator - def getBooksPage(self, bot, update): + @check_auth_decorator + def getBooksPage(self, update: Update, context: CallbackContext): callback_query = update.callback_query (query,page_num) = callback_query.data.split(query_delimiter, maxsplit=1) if (page_num == 'current'): @@ -229,14 +226,17 @@ def getBooksPage(self, bot, update): except: page_num = 1 - books = self.BookFilter(query) - response = self.BookPager(books, page_num, query) - bot.edit_message_text(chat_id=callback_query.message.chat_id, message_id=callback_query.message.message_id, text=response['message'], parse_mode='HTML', reply_markup=response['buttons']) - return + books = self.bookFilter(query) + response = self.bookPager(books, page_num, query) + try: + context.bot.edit_message_text(chat_id=callback_query.message.chat_id, message_id=callback_query.message.message_id, text=response['message'], parse_mode='HTML', reply_markup=response['buttons']) + except BadRequest: + pass + return None @cmdtrans - @CheckAuthDecorator - def downloadBooks(self, bot, update): + @check_auth_decorator + def downloadBooks(self, update: Update, context: CallbackContext): book_id_set=re.findall(r'\d+$',update.message.text) if len(book_id_set)==1: try: @@ -251,7 +251,7 @@ def downloadBooks(self, bot, update): if book==None: response = _("The book on the link you specified is not found, try to repeat the book search first.") - bot.sendMessage(chat_id=update.message.chat_id, text=response, parse_mode='HTML') + context.bot.sendMessage(chat_id=update.message.chat_id, text=response, parse_mode='HTML') self.logger.info("Not find download links: %s" % response) return @@ -259,7 +259,7 @@ def downloadBooks(self, bot, update): response = ('%(title)s\n%(author)s\n'+_("Annotation:")+'%(annotation)s\n') % {'title': book.title, 'author': authors, 'annotation':book.annotation} buttons = [InlineKeyboardButton(book.format.upper(), callback_data='/getfileorig%s'%book_id)] - if not book.format in settings.NOZIP_FORMATS: + if book.format not in settings.NOZIP_FORMATS: buttons += [InlineKeyboardButton(book.format.upper()+'.ZIP', callback_data='/getfilezip%s'%book_id)] if (config.SOPDS_FB2TOEPUB != "") and (book.format == 'fb2'): buttons += [InlineKeyboardButton('EPUB', callback_data='/getfileepub%s'%book_id)] @@ -267,13 +267,13 @@ def downloadBooks(self, bot, update): buttons += [InlineKeyboardButton('MOBI', callback_data='/getfilemobi%s'%book_id)] markup = InlineKeyboardMarkup([buttons]) - bot.sendMessage(chat_id=update.message.chat_id, text=response, parse_mode='HTML', reply_markup=markup) + context.bot.sendMessage(chat_id=update.message.chat_id, text=response, parse_mode='HTML', reply_markup=markup) self.logger.info("Send download buttons.") - return + return None @cmdtrans - @CheckAuthDecorator - def getBookFile(self, bot, update): + @check_auth_decorator + def getBookFile(self, update: Update, context: CallbackContext): callback_query = update.callback_query query = callback_query.data book_id_set=re.findall(r'\d+$',query) @@ -289,7 +289,7 @@ def getBookFile(self, bot, update): if book==None: response = _("The book on the link you specified is not found, try to repeat the book search first.") - bot.sendMessage(chat_id=callback_query.message.chat_id, text=response, parse_mode='HTML') + context.bot.sendMessage(chat_id=callback_query.message.chat_id, text=response, parse_mode='HTML') self.logger.info("Not find download links: %s" % response) return @@ -298,44 +298,39 @@ def getBookFile(self, bot, update): if re.match(r'/getfileorig',query): document = dl.getFileData(book) - #document = config.SOPDS_SITE_ROOT + reverse("opds_catalog:download",kwargs={"book_id": book.id, "zip_flag": 0}) if re.match(r'/getfilezip',query): document = dl.getFileDataZip(book) - #document = config.SOPDS_SITE_ROOT + reverse("opds_catalog:download", kwargs={"book_id": book.id, "zip_flag": 1}) filename = filename + '.zip' if re.match(r'/getfileepub',query): document = dl.getFileDataEpub(book) - #document = config.SOPDS_SITE_ROOT+reverse("opds_catalog:convert",kwargs={"book_id": book.id, "convert_type": "epub"}))] - filename = filename + '.epub' + filename = filename.replace('.fb2', '.epub') if re.match(r'/getfilemobi',query): document = dl.getFileDataMobi(book) - #document = config.SOPDS_SITE_ROOT+reverse("opds_catalog:convert",kwargs={"book_id": book.id, "convert_type": "mobi"}))] - filename = filename + '.mobi' + filename = filename.replace('.fb2', '.mobi') if document: - bot.send_document(chat_id=callback_query.message.chat_id,document=document,filename=filename) + context.bot.send_document(chat_id=callback_query.message.chat_id,document=document,filename=filename) document.close() self.logger.info("Send file: %s" % filename) else: response = _("There was a technical error, please contact the Bot administrator.") - bot.sendMessage(chat_id=callback_query.message.chat_id, text=response, parse_mode='HTML') + context.bot.sendMessage(chat_id=callback_query.message.chat_id, text=response, parse_mode='HTML') self.logger.info("Book get error: %s" % response) - return - return + return None @cmdtrans - @CheckAuthDecorator - def botCallback(self, bot, update): + @check_auth_decorator + def botCallback(self, update: Update, context: CallbackContext): query = update.callback_query if re.match(r'/getfile', query.data): - return self.getBookFile(bot, update) + return self.getBookFile(update, context) else: - return self.getBooksPage(bot, update) + return self.getBooksPage(update, context) def start(self): writepid(self.pidfile) @@ -344,32 +339,38 @@ def start(self): try: updater = Updater(token=config.SOPDS_TELEBOT_API_TOKEN) start_command_handler = CommandHandler('start', self.startCommand) - getBook_handler = MessageHandler(Filters.text, self.getBooks) - download_handler = RegexHandler('^/download\d+$',self.downloadBooks) + get_book_handler = MessageHandler(Filters.text, self.getBooks) + #fix deprecated RegexHandler See https://git.io/fxJuV for more info + download_handler = MessageHandler(Filters.regex('^/download\d+$'),self.downloadBooks) updater.dispatcher.add_handler(start_command_handler) - updater.dispatcher.add_handler(getBook_handler) + #change order of handlers, to handle download(regexp) before common text(book name) updater.dispatcher.add_handler(download_handler) + updater.dispatcher.add_handler(get_book_handler) updater.dispatcher.add_handler(CallbackQueryHandler(self.botCallback)) - updater.start_polling(clean=True) + updater.start_polling(drop_pending_updates=True) updater.idle() except InvalidToken: self.stdout.write('Invalid telegram token.\nSet correct token for telegram API by command:\n python3 manage.py sopds_util setconf SOPDS_TELEBOT_API_TOKEN ""') self.logger.error('Invalid telegram token.') except (KeyboardInterrupt, SystemExit): - pass - + pass + + return None + def stop(self, pid): try: os.kill(int(pid), signal.SIGTERM) except OSError as e: self.stdout.write("Error stopping sopds_telebot: %s"%str(e)) - + return None + def restart(self, pid): self.stop(pid) self.start() + return None def writepid(pid_file): """ @@ -378,7 +379,7 @@ def writepid(pid_file): fp = open(pid_file, "w") fp.write(str(os.getpid())) fp.close() - + def daemonize(): """ Detach from the terminal and continue as a daemon. @@ -396,14 +397,14 @@ def daemonize(): std_out = open(config.SOPDS_TELEBOT_LOG, 'a+') os.dup2(std_in.fileno(), sys.stdin.fileno()) os.dup2(std_out.fileno(), sys.stdout.fileno()) - os.dup2(std_out.fileno(), sys.stderr.fileno()) - + os.dup2(std_out.fileno(), sys.stderr.fileno()) + os.close(std_in.fileno()) os.close(std_out.fileno()) + return None + + - - - diff --git a/opds_catalog/middleware.py b/opds_catalog/middleware.py index e97d6733..8561eed8 100644 --- a/opds_catalog/middleware.py +++ b/opds_catalog/middleware.py @@ -35,7 +35,11 @@ def process_request(self,request): except KeyError: return self.unauthed() - (auth_meth, auth_data) = authentication.split(' ',1) + try: + (auth_meth, auth_data) = authentication.split(' ',1) + except ValueError: + return self.unauthed() + if 'basic' != auth_meth.lower(): return self.unauthed() auth_data = base64.b64decode(auth_data.strip()).decode('utf-8') @@ -45,7 +49,7 @@ def process_request(self,request): if user and user.is_active: request.user = user auth.login(request, user) - return None + return request return self.unauthed() diff --git a/sopds_web_backend/views.py b/sopds_web_backend/views.py index e1192f6f..8fa7a88a 100644 --- a/sopds_web_backend/views.py +++ b/sopds_web_backend/views.py @@ -241,7 +241,11 @@ def SearchBooksView(request): args['books']=items args['current'] = 'search' args['cache_id']='%s:%s:%s'%(searchterms,searchtype,op.page_num) - args['cache_t']=config.SOPDS_CACHE_TIME + # changes on bookshelf should be refreshed immediatelly + if searchtype == 'u': + args['cache_t'] = 0 + else: + args['cache_t'] = config.SOPDS_CACHE_TIME return render(request,'sopds_books.html', args)