-
Notifications
You must be signed in to change notification settings - Fork 26
/
debugger.py
59 lines (41 loc) · 1.88 KB
/
debugger.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import sys
from os import path
from typing import Any, Dict, Optional
src_path = path.dirname(path.abspath(__file__))
sys.path.insert(0, src_path)
__all__ = ["create_debugger", "create_pg_debugger", "create_ch_debugger"]
def create_debugger(sql_file_path: str, vars: Optional[Dict[str, Any]] = None, funcs: Optional[Dict[str, Any]] = None):
import os
import subprocess
spark_home = (
subprocess.check_output(["bash", "-c", "echo 'import os; print(os.environ[\"SPARK_HOME\"])' | pyspark"])
.decode("utf8")
.split("\n")
)
spark_home = [c.strip() for c in spark_home if c.strip()][0]
os.environ["SPARK_HOME"] = spark_home
import findspark
findspark.init()
from pyspark.sql import SparkSession
from easy_sql.sql_processor.backend import SparkBackend
spark = SparkSession.builder.enableHiveSupport().getOrCreate()
backend = SparkBackend(spark)
from easy_sql.sql_processor_debugger import SqlProcessorDebugger
debugger = SqlProcessorDebugger(sql_file_path, backend, vars, funcs)
return debugger
def create_pg_debugger(
sql_file_path: str, vars: Optional[Dict[str, Any]] = None, funcs: Optional[Dict[str, Any]] = None
):
from easy_sql.sql_processor.backend.rdb import RdbBackend
pg = RdbBackend("postgresql://postgres:123456@testpg:15432/postgres")
from easy_sql.sql_processor_debugger import SqlProcessorDebugger
debugger = SqlProcessorDebugger(sql_file_path, pg, vars, funcs)
return debugger
def create_ch_debugger(
sql_file_path: str, vars: Optional[Dict[str, Any]] = None, funcs: Optional[Dict[str, Any]] = None
):
from easy_sql.sql_processor.backend.rdb import RdbBackend
ch = RdbBackend("clickhouse+native://default@testch:30123")
from easy_sql.sql_processor_debugger import SqlProcessorDebugger
debugger = SqlProcessorDebugger(sql_file_path, ch, vars, funcs)
return debugger