From 4599c7796ff24fc86022a9f2eec045ad6b9ef75f Mon Sep 17 00:00:00 2001 From: John Xina Date: Tue, 11 Jul 2023 19:32:13 +0800 Subject: [PATCH] Initial Commit --- .gitignore | 4 + app.py | 69 ++++++++++++++++++ extra.py | 33 +++++++++ filters.py | 16 ++++ main.py | 156 +++++++++++++++++++++++++++++++++++++++ requirements.txt | 5 ++ static/css/main.css | 166 ++++++++++++++++++++++++++++++++++++++++++ templates/bar.html | 53 ++++++++++++++ templates/thread.html | 42 +++++++++++ 9 files changed, 544 insertions(+) create mode 100644 .gitignore create mode 100644 app.py create mode 100644 extra.py create mode 100644 filters.py create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 static/css/main.css create mode 100644 templates/bar.html create mode 100644 templates/thread.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7017857 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +venv/ +log/ +backup/ diff --git a/app.py b/app.py new file mode 100644 index 0000000..30e7533 --- /dev/null +++ b/app.py @@ -0,0 +1,69 @@ +import asyncio +import aiotieba + +from aioflask import Flask, render_template, request +from datetime import datetime + +from extra import * + +app = Flask(__name__) + +###################################################################### + +# Convert a timestamp to its simpliest readable date format. +@app.template_filter('simpledate') +def _jinja2_filter_simpledate(ts): + t = datetime.fromtimestamp(ts) + now = datetime.now() + + if t.date() == now.date(): + return t.strftime('%H:%m') + elif t.year == now.year: + return t.strftime('%m-%d') + else: + return t.strftime('%Y-%m-%d') + +# Convert a timestamp to a humand readable date format. +@app.template_filter('date') +def _jinja2_filter_datetime(ts, fmt='%Y年%m月%d日 %H点%m分'): + return datetime.fromtimestamp(ts).strftime(fmt) + +# Convert a integer to the one with separator like 1,000,000. +@app.template_filter('intsep') +def _jinja2_filter_intsep(i): + return f'{int(i):,}' + +# Reduce the text to a shorter form. +@app.template_filter('trim') +def _jinja2_filter_trim(text): + return text[:78] + '……' if len(text) > 78 else text + +###################################################################### + +@app.route('/p/') +async def thread_view(tid): + tid = int(tid) + pn = int(request.args.get('pn') or 1) + + async with aiotieba.Client() as tieba: + # Default to 15 posts per page, confirm to tieba.baidu.com + thread_info = await tieba.get_posts(tid, rn=15, pn=pn) + + for fragment in thread_info[0].contents: + print(fragment) + + return await render_template('thread.html', info=thread_info) + +@app.route('/f') +async def forum_view(): + fname = request.args['kw'] + pn = int(request.args.get('pn') or 1) + + async with aiotieba.Client() as tieba: + forum_info, threads = await asyncio.gather(find_tieba_info(fname), + tieba.get_threads(fname, rn=50, pn=pn)) + + return await render_template('bar.html', info=forum_info, threads=threads) + +if __name__ == '__main__': + app.run(debug=True) diff --git a/extra.py b/extra.py new file mode 100644 index 0000000..ad8ccf6 --- /dev/null +++ b/extra.py @@ -0,0 +1,33 @@ +'''Extra APIs''' + +import requests +import bs4 +import re + +async def find_tieba_info(tname): + """Get the tiebat avatar for the forum name. + + :param tname: the name of the target forum. + :returns: the internal ID of the corresponding avatar. + + """ + info = { 'name': tname } + + res = requests.get('https://tieba.baidu.com/f', params={'kw': tname}) + soup = bs4.BeautifulSoup(res.text, 'html.parser') + + elems = soup.select('#forum-card-head') + match = re.search(r'/(\w+)\.jpg', elems[0]['src']) + + info['avatar'] = match.group(1) + '.jpg' + + footer = soup.select('.th_footer_l')[0] + stat_elems = footer.findAll('span', {'class': 'red_text'}, recursive=False) + stats = list(map(lambda x: int(x.text), stat_elems)) + + info |= { 'topic': stats[0], 'thread': stats[1], 'member': stats[2] } + + slogan = soup.select('.card_slogan')[0] + info['desc'] = slogan.text + + return info diff --git a/filters.py b/filters.py new file mode 100644 index 0000000..923f92b --- /dev/null +++ b/filters.py @@ -0,0 +1,16 @@ +from datetime import datetime, timedelta + +# Convert a timestamp to a humand readable date format. +@app.template_filter('date') +def _jinja2_filter_datetime(ts, fmt='%Y年%m月%d日 %H点%m分'): + return datetime.fromtimestamp(ts).strftime(fmt) + +# Convert a integer to the one with separator like 1,000,000. +@app.template_filter('intsep') +def _jinja2_filter_intsep(i): + return f'{int(i):,}' + +# Convert a duration in seconds to human readable duration. +@app.template_filter('secdur') +def __jinja2_filter_secdur(delta_t): + return str(timedelta(seconds=int(delta_t))) diff --git a/main.py b/main.py new file mode 100644 index 0000000..2bf0811 --- /dev/null +++ b/main.py @@ -0,0 +1,156 @@ +import multiprocessing + +from app import app + +from urllib.parse import quote as urlquote, urlparse, urlunparse +from twisted.web.http import _QUEUED_SENTINEL, HTTPChannel, HTTPClient, Request +from twisted.web.resource import Resource +from twisted.web import proxy, server +from twisted.internet.protocol import ClientFactory +from twisted.internet import reactor, utils + +plain_cookies = {} + +################################################################################ +# Modified Dynamic Proxy (from twisted) +################################################################################ + +class ProxyClient(HTTPClient): + _finished = False + + def __init__(self, command, rest, version, headers, data, father): + self.father = father + self.command = command + self.rest = rest + if b"proxy-connection" in headers: + del headers[b"proxy-connection"] + headers[b"connection"] = b"close" + headers.pop(b"keep-alive", None) + self.headers = headers + self.data = data + + def connectionMade(self): + self.sendCommand(self.command, self.rest) + for header, value in self.headers.items(): + self.sendHeader(header, value) + self.endHeaders() + self.transport.write(self.data) + + def handleStatus(self, version, code, message): + self.father.setResponseCode(int(code), message) + + def handleHeader(self, key, value): + if key.lower() in [b"server", b"date", b"content-type"]: + self.father.responseHeaders.setRawHeaders(key, [value]) + else: + self.father.responseHeaders.addRawHeader(key, value) + + def handleResponsePart(self, buffer): + self.father.write(buffer) + + def handleResponseEnd(self): + if not self._finished: + self._finished = True + self.father.notifyFinish().addErrback(lambda x: None) + self.transport.loseConnection() + +class ProxyClientFactory(ClientFactory): + protocol = ProxyClient + + def __init__(self, command, rest, version, headers, data, father): + self.father = father + self.command = command + self.rest = rest + self.headers = headers + self.data = data + self.version = version + + def buildProtocol(self, addr): + return self.protocol( + self.command, self.rest, self.version, self.headers, self.data, self.father + ) + + def clientConnectionFailed(self, connector, reason): + self.father.setResponseCode(501, b"Gateway error") + self.father.responseHeaders.addRawHeader(b"Content-Type", b"text/html") + self.father.write(b"

Could not connect

") + self.father.finish() + +class ReverseProxyResource(Resource): + def __init__(self, path, reactor=reactor): + Resource.__init__(self) + self.path = path + self.reactor = reactor + + def getChild(self, path, request): + return ReverseProxyResource( + self.path + b'/' + urlquote(path, safe=b'').encode("utf-8"), + self.reactor + ) + + def render_proxy_avatar(self, request, req_path): + portrait = req_path[14:] + + request.requestHeaders.setRawHeaders(b'host', [b'tb.himg.baidu.com']) + request.content.seek(0, 0) + + clientFactory = ProxyClientFactory( + b'GET', ('http://tb.himg.baidu.com/sys/portraith/item/' + portrait).encode('utf-8'), + request.clientproto, + request.getAllHeaders(), + request.content.read(), + request, + ) + + self.reactor.connectTCP('tb.himg.baidu.com', 80, clientFactory) + return server.NOT_DONE_YET + + def render_proxy_pic(self, request, req_path): + pic = req_path[11:] + + request.requestHeaders.setRawHeaders(b'host', [b'imgsa.baidu.com']) + request.content.seek(0, 0) + + clientFactory = ProxyClientFactory( + b'GET', ('http://imgsa.baidu.com/forum/pic/item/' + pic).encode('utf-8'), + request.clientproto, + request.getAllHeaders(), + request.content.read(), + request, + ) + + self.reactor.connectTCP('imgsa.baidu.com', 80, clientFactory) + return server.NOT_DONE_YET + + def render(self, request): + # Justify the request path. + req_path = self.path.decode('utf-8') + if req_path.startswith('/proxy/avatar/'): + return self.render_proxy_avatar(request, req_path) + elif req_path.startswith('/proxy/pic/'): + return self.render_proxy_pic(request, req_path) + else: + request.setResponseCode(418, b'I\'m a teapot') + return + +################################################################################ + +# To start this function for testing: python -c 'import main; main.twisted_start()' +def twisted_start(): + flask_res = proxy.ReverseProxyResource('127.0.0.1', 5000, b'') + flask_res.putChild(b'proxy', ReverseProxyResource(b'/proxy')) + + site = server.Site(flask_res) + reactor.listenTCP(5001, site) + reactor.run() + +# To start this function for testing: python -c 'import main; main.flask_start()' +def flask_start(): + app.run(port=5000+1) + +# If we're executed directly, also start the flask daemon. +if __name__ == '__main__': + flask_task = multiprocessing.Process(target=flask_start) + flask_task.daemon = True # Exit the child if the parent was killed :-( + flask_task.start() + twisted_start() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..72c0821 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +aioflask==0.4.0 +flask==2.1.3 +aiotieba==3.4.5 +beautifulsoup4 +requests diff --git a/static/css/main.css b/static/css/main.css new file mode 100644 index 0000000..4c87201 --- /dev/null +++ b/static/css/main.css @@ -0,0 +1,166 @@ +:root { + --bg-color: #eeeecc; + --fg-color: #ffffdd; + --replies-color: #f0f0d0; + --text-color: #663333; + --primary-color: #0066cc; + --important-color: #ff0000; +} + + +@media (prefers-color-scheme: dark) { + :root { + --bg-color: #000033; + --fg-color: #202044; + --replies-color: #16163a; + --text-color: #cccccc; + --primary-color: #6699ff; + --important-color: #ff0000; + } +} + +body { + max-width: 48rem; + margin: auto; + background-color: var(--bg-color); + color: var(--text-color); + font-family: sans-serif; +} + +a[href] { + color: var(--primary-color); + text-decoration: none; +} + +a[href]:hover { + text-decoration: underline; +} + +img { + max-width: 100%; +} + +.bar-nav, .thread-nav { + background-color: var(--fg-color); + display: flex; + flex-wrap: wrap; + gap: 0 1rem; + padding: 1rem; + margin-bottom: 1rem; + align-items: center; +} + +.thread-nav > .title { + font-size: 1.2rem; + flex: 1 0 70%; +} + +.thread-nav > .from { + font-size: 1.2rem; +} + +.bar-nav > img { + width: 5rem; + height: 5rem; +} + +.bar-nav .title { + font-size: 1.5rem; +} + +.bar-nav .stats small { + margin-right: .5rem; +} + +.list { + background-color: var(--fg-color); + margin-bottom: 1rem; +} + +.post { + display: flex; + flex-wrap: wrap; + gap: 0 1rem; + border-bottom: 1px solid var(--text-color); + padding: 1rem; +} + +.post .avatar { + width: 4rem; + height: 4rem; +} + +.post > div { + flex: 1; +} + +.post .userinfo { + display: flex; + gap: .5rem; + align-items: center; +} + +.tag { + border-radius: .4rem; + font-size: .8rem; + padding: .1rem .4rem; + margin-right: .2rem; + color: white; +} + +.tag-blue { + background-color: var(--primary-color); +} + +.tag-red { + background-color: var(--important-color); +} + +.post .permalink { + float: right; +} + +.thread { + display: flex; + gap: 1rem; + padding: 1rem; +} + +.thread .stats { + flex: 0 0 4rem; + text-align: center; +} + +.thread .replies { + font-size: .8rem; + padding: .5rem .2rem; + background-color: var(--replies-color); +} + +.thread .summary { + flex: 1 1 auto; +} + +.thread .title { + font-size: 1.1rem; + margin-bottom: .5rem; +} + +.thread .participants { + font-size: .8rem; + flex: 0 0 6rem; +} + +.thread .participants a { + padding-left: .3rem; + color: var(--text-color); +} + +footer { + display: flex; + gap: 2rem; + padding: 1rem; + justify-content: center; + align-items: center; + flex-wrap: wrap; +} diff --git a/templates/bar.html b/templates/bar.html new file mode 100644 index 0000000..c2f0ae7 --- /dev/null +++ b/templates/bar.html @@ -0,0 +1,53 @@ + + + {{ info['name'] }}吧 - RAT + + + + + + + + +
+ +
+
{{ info['name'] }}吧
+
{{ info['desc'] }}
+
+ 关注: {{ info['member']|intsep }} + 主题: {{ info['topic']|intsep }} + 帖子: {{ info['thread']|intsep }} +
+
+
+
+ {% for t in threads %} +
+
+
{{ t.reply_num }}
+ {{ t.last_time|simpledate }} +
+
+
+ {% if t.is_top or t.is_livepost + %}置顶{% + endif %}{% if t.is_good + %}精{% endif + %}{{ t.title }} +
+
{{ t.text[(t.title|length):]|trim }}
+
+
+ + +
+
+ {% endfor %} +
+ + diff --git a/templates/thread.html b/templates/thread.html new file mode 100644 index 0000000..7329fbf --- /dev/null +++ b/templates/thread.html @@ -0,0 +1,42 @@ + + + {{ info.thread.text }} - 自由软件吧 - RAT + + + + + + + + + +
+
{{ info.thread.title }}
+
{{ info.forum.fname }}吧
+
+
+ {% for p in info %} +
+ +
+
+ {{ p.user.user_name }} + {% if p.is_thread_author %} + 楼主 + {% endif %} +
+
+ {{ p['text'] }} +
+ {{ p.create_time|date }} + {{ p.floor }} +
+
+ {% endfor %} +
+ +