Prepare mobile de parser preview
This commit is contained in:
qananasikq
317
iaai_scraper/proxy_bridge.py
Normal file
317
iaai_scraper/proxy_bridge.py
Normal file
@@ -0,0 +1,317 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import select
|
||||
import socket
|
||||
import socketserver
|
||||
import struct
|
||||
import threading
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
BUFFER_SIZE = 65536
|
||||
CRLF = b"\r\n"
|
||||
DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
|
||||
DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
|
||||
SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
|
||||
SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
|
||||
SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
|
||||
SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
|
||||
RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
|
||||
MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
|
||||
|
||||
logger = logging.getLogger("proxy_bridge")
|
||||
|
||||
|
||||
class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, server_address, request_handler_class):
|
||||
super().__init__(server_address, request_handler_class)
|
||||
self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
|
||||
|
||||
def process_request_thread(self, request, client_address):
|
||||
with self._worker_semaphore:
|
||||
super().process_request_thread(request, client_address)
|
||||
|
||||
|
||||
def _recv_exact(sock: socket.socket, size: int) -> bytes:
|
||||
data = b""
|
||||
while len(data) < size:
|
||||
chunk = sock.recv(size - len(data))
|
||||
if not chunk:
|
||||
raise ConnectionError("Unexpected EOF from SOCKS5 server")
|
||||
data += chunk
|
||||
return data
|
||||
|
||||
|
||||
def _socks5_connect(host: str, port: int) -> socket.socket:
|
||||
if not SOCKS5_HOST:
|
||||
raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
|
||||
|
||||
upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
|
||||
upstream.settimeout(30)
|
||||
|
||||
methods = [0x00]
|
||||
if SOCKS5_USER or SOCKS5_PASS:
|
||||
methods = [0x02]
|
||||
upstream.sendall(bytes([0x05, len(methods), *methods]))
|
||||
version, method = _recv_exact(upstream, 2)
|
||||
if version != 0x05 or method == 0xFF:
|
||||
upstream.close()
|
||||
raise ConnectionError("SOCKS5 authentication negotiation failed")
|
||||
|
||||
if method == 0x02:
|
||||
username = SOCKS5_USER.encode("utf-8")
|
||||
password = SOCKS5_PASS.encode("utf-8")
|
||||
if len(username) > 255 or len(password) > 255:
|
||||
upstream.close()
|
||||
raise ValueError("SOCKS5 username/password too long")
|
||||
upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
|
||||
auth_version, auth_status = _recv_exact(upstream, 2)
|
||||
if auth_version != 0x01 or auth_status != 0x00:
|
||||
upstream.close()
|
||||
raise ConnectionError("SOCKS5 username/password authentication failed")
|
||||
|
||||
try:
|
||||
socket.inet_aton(host)
|
||||
addr_type = 0x01
|
||||
addr_payload = socket.inet_aton(host)
|
||||
except OSError:
|
||||
host_bytes = host.encode("idna")
|
||||
if len(host_bytes) > 255:
|
||||
upstream.close()
|
||||
raise ValueError("Target host is too long for SOCKS5 domain format")
|
||||
addr_type = 0x03
|
||||
addr_payload = bytes([len(host_bytes)]) + host_bytes
|
||||
|
||||
request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
|
||||
upstream.sendall(request)
|
||||
|
||||
response_head = _recv_exact(upstream, 4)
|
||||
version, reply, _reserved, reply_addr_type = response_head
|
||||
if version != 0x05 or reply != 0x00:
|
||||
upstream.close()
|
||||
raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
|
||||
|
||||
if reply_addr_type == 0x01:
|
||||
_recv_exact(upstream, 4)
|
||||
elif reply_addr_type == 0x03:
|
||||
domain_len = _recv_exact(upstream, 1)[0]
|
||||
_recv_exact(upstream, domain_len)
|
||||
elif reply_addr_type == 0x04:
|
||||
_recv_exact(upstream, 16)
|
||||
_recv_exact(upstream, 2)
|
||||
|
||||
upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
return upstream
|
||||
|
||||
|
||||
def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
|
||||
sockets = [left, right]
|
||||
left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
try:
|
||||
while True:
|
||||
readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
if exceptional:
|
||||
break
|
||||
if not readable:
|
||||
logger.debug("Relay idle timeout reached; closing sockets")
|
||||
return
|
||||
for current in readable:
|
||||
other = right if current is left else left
|
||||
data = current.recv(BUFFER_SIZE)
|
||||
if not data:
|
||||
return
|
||||
other.sendall(data)
|
||||
finally:
|
||||
for sock in sockets:
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class ProxyHandler(socketserver.StreamRequestHandler):
|
||||
def handle(self) -> None:
|
||||
try:
|
||||
request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
|
||||
if not request_line:
|
||||
return
|
||||
|
||||
method, target, version = request_line.split()
|
||||
headers = self._read_headers()
|
||||
logger.info("%s %s", method, target)
|
||||
|
||||
if method.upper() == "CONNECT":
|
||||
host, port = self._parse_connect_target(target)
|
||||
logger.info("CONNECT %s:%s", host, port)
|
||||
upstream = _socks5_connect(host, port)
|
||||
self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
|
||||
self.wfile.flush()
|
||||
_relay_bidirectional(self.connection, upstream)
|
||||
return
|
||||
|
||||
host, port, path = self._parse_forward_target(target, headers)
|
||||
upstream = _socks5_connect(host, port)
|
||||
self._send_forward_request(upstream, method, path, version, headers)
|
||||
body = self._read_request_body(headers)
|
||||
if body:
|
||||
upstream.sendall(body)
|
||||
_relay_bidirectional(self.connection, upstream)
|
||||
except Exception as exc:
|
||||
logger.exception("Proxy bridge request failed: %s", exc)
|
||||
try:
|
||||
self.wfile.write(
|
||||
b"HTTP/1.1 502 Bad Gateway" + CRLF
|
||||
+ b"Connection: close" + CRLF
|
||||
+ b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
|
||||
+ b"Bad Gateway"
|
||||
)
|
||||
self.wfile.flush()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _read_headers(self) -> list[tuple[str, str]]:
|
||||
headers: list[tuple[str, str]] = []
|
||||
while True:
|
||||
line = self.rfile.readline(BUFFER_SIZE)
|
||||
if line in {CRLF, b"\n", b""}:
|
||||
break
|
||||
decoded = line.decode("iso-8859-1")
|
||||
if ":" not in decoded:
|
||||
continue
|
||||
name, value = decoded.split(":", 1)
|
||||
headers.append((name.strip(), value.strip()))
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def _parse_connect_target(target: str) -> tuple[str, int]:
|
||||
if target.startswith("["):
|
||||
end = target.find("]")
|
||||
if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
|
||||
raise ValueError("Invalid CONNECT target")
|
||||
host = target[1:end]
|
||||
port_text = target[end + 2 :]
|
||||
return host, int(port_text)
|
||||
|
||||
host, port_text = target.rsplit(":", 1)
|
||||
return host, int(port_text)
|
||||
|
||||
@staticmethod
|
||||
def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
|
||||
if target.startswith("http://"):
|
||||
parts = urlsplit(target)
|
||||
port = parts.port or 80
|
||||
path = parts.path or "/"
|
||||
if parts.query:
|
||||
path += f"?{parts.query}"
|
||||
return parts.hostname or "", port, path
|
||||
|
||||
if target.startswith("https://"):
|
||||
raise ValueError("HTTPS absolute-form request must use CONNECT")
|
||||
|
||||
host_header = next((value for name, value in headers if name.lower() == "host"), "")
|
||||
if not host_header:
|
||||
raise ValueError("Missing Host header")
|
||||
if ":" in host_header:
|
||||
host, port_text = host_header.rsplit(":", 1)
|
||||
return host, int(port_text), target
|
||||
return host_header, 80, target
|
||||
|
||||
def _send_forward_request(
|
||||
self,
|
||||
upstream: socket.socket,
|
||||
method: str,
|
||||
path: str,
|
||||
version: str,
|
||||
headers: list[tuple[str, str]],
|
||||
) -> None:
|
||||
filtered_headers: list[tuple[str, str]] = []
|
||||
hop_by_hop = {
|
||||
"proxy-connection",
|
||||
"proxy-authorization",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
for name, value in headers:
|
||||
if name.lower() in hop_by_hop:
|
||||
continue
|
||||
filtered_headers.append((name, value))
|
||||
|
||||
request_head = [f"{method} {path} {version}\r\n"]
|
||||
request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
|
||||
request_head.append("\r\n")
|
||||
upstream.sendall("".join(request_head).encode("iso-8859-1"))
|
||||
|
||||
def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
|
||||
transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
|
||||
if "chunked" in transfer_encoding.lower():
|
||||
return self._read_chunked_request_body()
|
||||
|
||||
content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
|
||||
if not content_length:
|
||||
return b""
|
||||
return self.rfile.read(int(content_length))
|
||||
|
||||
def _read_chunked_request_body(self) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
while True:
|
||||
size_line = self.rfile.readline(BUFFER_SIZE)
|
||||
if not size_line:
|
||||
raise ConnectionError("Unexpected EOF in chunked request")
|
||||
size_text = size_line.strip().split(b";", 1)[0]
|
||||
chunk_size = int(size_text, 16)
|
||||
chunks.append(size_line)
|
||||
if chunk_size == 0:
|
||||
while True:
|
||||
trailer_line = self.rfile.readline(BUFFER_SIZE)
|
||||
if not trailer_line:
|
||||
raise ConnectionError("Unexpected EOF in chunked trailers")
|
||||
chunks.append(trailer_line)
|
||||
if trailer_line in {CRLF, b"\n"}:
|
||||
return b"".join(chunks)
|
||||
|
||||
chunk_data = self.rfile.read(chunk_size)
|
||||
if len(chunk_data) != chunk_size:
|
||||
raise ConnectionError("Unexpected EOF in chunk body")
|
||||
chunks.append(chunk_data)
|
||||
chunk_end = self.rfile.read(2)
|
||||
if chunk_end != CRLF:
|
||||
raise ConnectionError("Invalid chunk terminator")
|
||||
chunks.append(chunk_end)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not SOCKS5_HOST:
|
||||
raise SystemExit("SOCKS5_PROXY_HOST is required")
|
||||
|
||||
logging.basicConfig(
|
||||
level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
|
||||
format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
|
||||
)
|
||||
|
||||
with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
|
||||
logger.info(
|
||||
"Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
|
||||
DEFAULT_LISTEN_HOST,
|
||||
DEFAULT_LISTEN_PORT,
|
||||
SOCKS5_HOST,
|
||||
SOCKS5_PORT,
|
||||
MAX_WORKERS,
|
||||
)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user