from pyspark.sql import SparkSession
from pytispark.pytispark import TiContext
spark = SparkSession.builder.appName("test") \
.master("spark://172.18.0.12:7077") \
.config("spark.tispark.pd.addresses", "172.18.0.12:2379") \
.getOrCreate()
ti = TiContext(spark)
2020-08-09 14:56:28 WARN Utils:66 - Set SPARK_LOCAL_IP if you need to bind to another address
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by org.apache.hadoop.security.authentication.util.KerberosUtil (file:/home/caikunling/.local/lib/python3.6/site-packages/pyspark/jars/hadoop-auth-2.7.3.jar) to method sun.security.krb5.Config.getInstance()
WARNING: Please consider reporting this to the maintainers of org.apache.hadoop.security.authentication.util.KerberosUtil
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
2020-08-09 14:56:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform… using builtin-java classes where applicable
Setting default log level to “WARN”.
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Traceback (most recent call last):
File “spark_select_test.py”, line 222, in
ti = TiContext(spark)
File “local/lib/python3.6/site-packages/pytispark/pytispark.py”, line 31, in init
self.ti = gw.jvm.TiExtensions.getInstance(sparkSession._jsparkSession).getOrCreateTiContext(sparkSession._jsparkSession)
TypeError: ‘JavaPackage’ object is not callable
Python 3.6.9 (default, Jul 17 2020, 12:50:27)
[GCC 8.4.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
20/08/14 01:01:43 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/__ / .__/\_,_/_/ /_/\_\ version 2.4.6
/_/
Using Python version 3.6.9 (default, Jul 17 2020 12:50:27)
SparkSession available as 'spark'.
>>> import pytispark
>>> import pytispark.pytispark as pti
>>> from pyspark.sql import SparkSession
>>> spark = SparkSession.builder.getOrCreate()
>>> ti = pti.TiContext(spark)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/tidb/.local/lib/python3.6/site-packages/pytispark/pytispark.py", line 31, in __init__
self.ti = gw.jvm.TiExtensions.getInstance(sparkSession._jsparkSession).getOrCreateTiContext(sparkSession._jsparkSession)
File "/home/tidb/spark-2.4.6-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1487, in __getattr__
py4j.protocol.Py4JError: org.apache.spark.sql.TiExtensions.getInstance does not exist in the JVM
>>>
20/08/14 12:41:56 WARN SparkContext: Using an existing SparkContext; some configuration may not take effect. [3/3470]
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/__ / .__/\_,_/_/ /_/\_\ version 2.4.6
/_/
Using Python version 3.6.9 (default, Jul 17 2020 12:50:27)
SparkSession available as 'spark'.
>>> sql('show databases')
20/08/14 12:42:10 INFO ReflectionUtil$: tispark class url: file:/home/caikunling/spark-2.4.6-bin-hadoop2.7/jars/tispark-assembly-2.3.1.jar
20/08/14 12:42:10 INFO ReflectionUtil$: spark wrapper class url: jar:file:/home/caikunling/spark-2.4.6-bin-hadoop2.7/jars/tispark-assembly-2.3.1.jar!/resources/spark-wrapper-spark-2.4/
DataFrame[databaseName: string]
>>> sql('show databases').show()
+-----------------+
| databaseName|
+-----------------+
| default|
|face_world_office|
| test|
| tpch_001|
| mysql|
+-----------------+
>>>
[caikunling@FACE_DB ~]20-08-14 12:42$ ipython
Python 3.6.9 (default, Jul 17 2020, 12:50:27)
Type 'copyright', 'credits' or 'license' for more information
IPython 7.16.1 -- An enhanced Interactive Python. Type '?' for help.
In [1]: from pyspark.sql import SparkSession
In [2]: spark = SparkSession.builder.appName("tidb-test") \
...: .master("spark://127.0.0.1:7077") \
...: .config("spark.tispark.pd.addresses", "127.0.0.1:2379") \
...: .config('spark.sql.extensions','org.apache.spark.sql.TiExtensions') \
...: .getOrCreate()
20/08/14 12:42:36 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
In [3]: spark.sql('show databases').show()
+------------+
|databaseName|
+------------+
| default|
+------------+
–jars ~/spark-2.4.6-bin-hadoop2.7/jars/tispark-assembly-2.3.1.jar
~/face-world-demo/spark_select_test.py
20/08/14 13:01:51 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform… using builtin-java classes where applicable
Traceback (most recent call last):
File “/home/tidb/face-world-demo/spark_select_test.py”, line 224, in
ti = TiContext(spark)
File “/home/tidb/.local/lib/python3.6/site-packages/pytispark/pytispark.py”, line 31, in init
self.ti = gw.jvm.TiExtensions.getInstance(sparkSession._jsparkSession).getOrCreateTiContext(sparkSession._jsparkSession)
File “/home/tidb/spark-2.4.6-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py”, line 1487, in getattr
py4j.protocol.Py4JError: org.apache.spark.sql.TiExtensions.getInstance does not exist in the JVM
