org.python.core.PySystemState Java Examples

/**
 * Complete package name
 *
 * @param target Target
 * @return Package names
 */
public List<String> completePackageName(String target) {
    String[] targetComponents = target.split("\\.");
    String base = targetComponents[0];
    PySystemState state = interp.getSystemState();
    PyObject importer = state.getBuiltins().__getitem__(Py.newString("__import__"));
    PyObject module = importer.__call__(Py.newString(base));
    if (targetComponents.length > 1) {
        for (int i = 1; i < targetComponents.length; i++) {
            module = module.__getattr__(targetComponents[i]);
        }
    }
    PyList plist = (PyList) module.__dir__();
    List<String> list = new ArrayList<>();
    String name;
    for (int i = 0; i < plist.__len__(); i++) {
        name = plist.get(i).toString();
        if (!name.startsWith("__")) {
            list.add(name);
        }
    }
    //list.add("*");

    return list;
}
 

@Override
public void initialize(UimaContext context)
		throws ResourceInitializationException {
	super.initialize(context);

	interp = new PythonInterpreter(null, new PySystemState());
	// LATER allow for nonstandard Python modules
	// PySystemState sys = Py.getSystemState();
	// sys.path.append(new PyString(rootPath));
	// sys.path.append(new PyString(modulesDir));

	try {
		scriptFileIs = ResourceHelper.getInputStream(scriptPath);
	} catch (Exception e) {
		throw new ResourceInitializationException(
				ResourceInitializationException.NO_RESOURCE_FOR_PARAMETERS,
				new Object[] { scriptPath });
	}
}
 

private static synchronized ScriptEngine getScriptEngine() throws ScriptException {
    if (scriptEngine == null) {
        PySystemState engineSys = new PySystemState();
        String assetRoot = ApplicationContext.getAssetRoot().getAbsolutePath();
        engineSys.path.append(Py.newString(assetRoot));
        Py.setSystemState(engineSys);
        scriptEngine = new PyScriptEngineFactory().getScriptEngine();
        // custom module finder for assets
        String assetPath = "com.centurylink.mdw.python/ModuleFinder.py";
        try {
            Asset finderAsset = AssetCache.getAsset(assetPath);
            Map<String, Object> values = new HashMap<>();
            values.put("assetRoot", assetRoot);
            Bindings bindings = new SimpleBindings(values);
            ScriptContext scriptContext = new SimpleScriptContext();
            scriptContext.setBindings(bindings, ScriptContext.ENGINE_SCOPE);
            scriptEngine.eval(finderAsset.getText(), scriptContext);
        } catch (IOException ex) {
            throw new ScriptException(ex);
        }
    }
    return scriptEngine;
}
 

/** Initialize
 *  @throws Exception on error
 */
public JythonSupport() throws Exception
{
    final PySystemState state = new PySystemState();

    // Creating a PythonInterpreter is very slow.
    //
    // In addition, concurrent creation is not supported, resulting in
    //     Lib/site.py", line 571, in <module> ..
    //     Lib/sysconfig.py", line 159, in _subst_vars AttributeError: {'userbase'}
    // or  Lib/site.py", line 122, in removeduppaths java.util.ConcurrentModificationException
    //
    // Sync. on JythonSupport to serialize the interpreter creation and avoid above errors.
    // Curiously, this speeds the interpreter creation up,
    // presumably because they're not concurrently trying to access the same resources?
    synchronized (JythonSupport.class)
    {
        interpreter = new PythonInterpreter(null, state);
    }
}
 

private PythonInterpreter createInterpreter()
{
    // System.out.println("Creating interpreter on " + Thread.currentThread().getName());

    final PySystemState state = new PySystemState();
    // Check for default settings
    if (! state.path.toString().contains("always"))
    {
        System.out.println("Not running in fresh VM, missing /tmp/always in " + state.path);
        state.path.add(0, "/tmp/always");
    }
    assertThat(state.path.toString(), not(containsString("special")));
    // Add settings specific to this interpreter
    state.path.add(0, "/tmp/special");
    return new PythonInterpreter(null, state);
}
 

public void getSysPath() {
  synchronized (sysPath) {
    if (null == interpreter) {
      return;
    }
    sysPath.clear();
    try {
      PySystemState pyState = interpreter.getSystemState();
      PyList pyPath = pyState.path;
      int pathLen = pyPath.__len__();
      for (int i = 0; i < pathLen; i++) {
        String entry = (String) pyPath.get(i);
        log(lvl + 1, "sys.path[%2d] = %s", i, entry);
        sysPath.add(entry);
      }
    } catch (Exception ex) {
      sysPath.clear();
    }
  }
}
 

private static synchronized PythonInterpreter initPythonInterpreter(String[] args, String pythonPath, String scriptName) {
	if (!jythonInitialized) {
		// the java stack traces within the jython runtime aren't useful for users
		System.getProperties().put("python.options.includeJavaStackInExceptions", "false");
		PySystemState.initialize(System.getProperties(), new Properties(), args);

		pythonInterpreter = new PythonInterpreter();

		pythonInterpreter.getSystemState().path.add(0, pythonPath);

		pythonInterpreter.setErr(System.err);
		pythonInterpreter.setOut(System.out);

		pythonInterpreter.exec("import " + scriptName);
		jythonInitialized = true;
	}
	return pythonInterpreter;
}
 

private boolean keyIsExcluded(String key, PyObject value) {
    return (key.startsWith("__") && key.endsWith("__")) ||
            value instanceof PyFile ||
            value instanceof PyModule ||
            value instanceof PyFunction ||
            value instanceof PySystemState ||
            value instanceof PyClass ||
            value instanceof PyType ||
            value instanceof PyReflectedFunction;
}
 

private void importPackage(String script) {
Matcher matcher = fromRegExp.matcher(script);
while (matcher.find()) {
	String packageName = matcher.group(1);
	PySystemState.add_package(packageName);
}
 }
 

public static PySystemState getPythonSystemState() {
	PySystemState sys = new PySystemState();
	addPathToPySystemState(sys, "./");
	addPathToPySystemState(sys, "./python/");
	addPathToPySystemState(sys, "./python-plugins/");
	File dependencyDirectory = new File("./");
	File[] files = dependencyDirectory.listFiles();
	for (int i = 0; i < files.length; i++) {
	    if (files[i].getName().toLowerCase().contains("spigot") && files[i].getName().toLowerCase().endsWith(".jar")) {
	    	addPathToPySystemState(sys, files[i].getAbsolutePath());
	    }
	}
	return sys;
}
 

public static PySystemState createPySystemState() {
    try {
        return new PySystemState();
    } catch (IllegalStateException e) {
        //happens when running tests.
        PySystemState.initialize();
        return new PySystemState();
    }
}
 

private static void setupJython() {
    synchronized (lock) {
        if (bundles != null && plugin != null) {
            //initialize the Jython runtime
            Properties prop2 = new Properties();
            prop2.put("python.home", FileUtils.getFileAbsolutePath(plugin.getPluginRootDir()));
            prop2.put("python.path", FileUtils.getFileAbsolutePath(getJySrcDirFile()));
            prop2.put("python.console.encoding", "UTF-8"); // Used to prevent: console: Failed to install '': java.nio.charset.UnsupportedCharsetException: cp0.
            prop2.put("python.security.respectJavaAccessibility", "false"); //don't respect java accessibility, so that we can access protected members on subclasses
            // prop2.put("python.verbose", "0"); //Don't print anything -- Jython itself won't start!
            try {
                AllBundleClassLoader allBundleClassLoader = new AllBundleClassLoader(plugin.getClass()
                        .getClassLoader());

                PySystemState.initialize(System.getProperties(), prop2, new String[0], allBundleClassLoader);
                List<String> packageNames = allBundleClassLoader.setBundlesAndGetPackageNames(bundles);
                int size = packageNames.size();
                for (int i = 0; i < size; ++i) {
                    String name = packageNames.get(i);
                    if (name.contains("internal")) {
                        continue;
                    }
                    int iToSplit = name.indexOf(';');
                    if (iToSplit != -1) {
                        name = name.substring(0, iToSplit);
                    }
                    //System.out.println("Added: " + name);
                    PySystemState.add_package(name);
                }
            } catch (Exception e) {
                Log.log(e);
            } finally {
                bundles = null;
            }
        }
    }
}
 

@Override
protected Map<String, List<PigStats>> main(PigContext pigContext, String scriptFile)
        throws IOException {
    if (System.getProperty(PySystemState.PYTHON_CACHEDIR_SKIP)==null)
        System.setProperty(PySystemState.PYTHON_CACHEDIR_SKIP, "false");
    
    PigServer pigServer = new PigServer(pigContext, false);

    // register dependencies
    String jythonJar = getJarPath(PythonInterpreter.class);
    if (jythonJar != null) {
        pigServer.registerJar(jythonJar);
    }

    File f = new File(scriptFile);

    if (!f.canRead()) {
        throw new IOException("Can't read file: " + scriptFile);
    }

    FileInputStream fis1 = new FileInputStream(scriptFile);
    try {
        if (hasFunction(fis1)) {
            registerFunctions(scriptFile, null, pigContext);
        }
    } finally {
        fis1.close();
    }

    Interpreter.setMain(true);
    FileInputStream fis = new FileInputStream(scriptFile);
    try {
        load(fis, scriptFile, pigServer.getPigContext());
    } finally {
        fis.close();
    }
    return getPigStatsMap();
}
 

protected PythonInterpreter initInterpreter(Set<String> dependencies) {
    logger.info("Creating python interpreter with [" + dependencies.size() + "] dependencies [" + dependencies + "]");
    if(!dependencies.isEmpty()) {
        PySystemState systemState = new PySystemState();
        for (String dependency: dependencies) {
            systemState.path.append(new PyString(dependency));
        }
        return new ThreadSafePythonInterpreter(systemState);
    }
    return GLOBAL_INTERPRETER;
}
 

private PythonInterpreter createInterpreterWithArgs(String xmlReportFile, String ignoreFile)
{
    PySystemState systemState = new PySystemState();

    if (xmlReportFile != null)
    {
        systemState.argv.append(new PyString("--xml"));
        systemState.argv.append(new PyString(xmlReportFile));
    }

    if(ignoreFile == null)
    {
        ignoreFile = System.getProperty(IGNORE_FILE_SYSTEM_PROPERTY);
    }

    if(ignoreFile != null)
    {
        systemState.argv.append(new PyString("-I"));
        systemState.argv.append(new PyString(ignoreFile));
    }

    String testPattern = System.getProperty(TEST_PATTERN_SYSTEM_PROPERTY);
    if(testPattern != null)
    {
        systemState.argv.append(new PyString(testPattern));
    }

    if(Boolean.getBoolean(ALWAYS_COLORIZE_SYSTEM_PROPERTY))
    {
        systemState.argv.append(new PyString("--always-colorize"));
    }

    PythonInterpreter interp = new PythonInterpreter(null, systemState);
    return interp;
}
 

private static void runScript(String args[], String fn, int idx) {
    String ext = GlobalUtil.getFileExtension(fn);
    System.out.println("Running Jython script...");
    //PySystemState state = Py.getSystemState();
    //Py.getSystemState().setdefaultencoding("utf-8");
    PySystemState state = new PySystemState();
    //state.setdefaultencoding("utf-8");
    if (args.length > idx + 1) {
        for (int i = idx + 1; i < args.length; i++) {
            state.argv.append(new PyString(args[i]));
        }
    }

    PythonInterpreter interp = new PythonInterpreter(null, state);
    String pluginPath = GlobalUtil.getAppPath(FrmMain.class) + File.separator + "plugins";
    List<String> jarfns = GlobalUtil.getFiles(pluginPath, ".jar");
    String path = GlobalUtil.getAppPath(FrmMain.class) + File.separator + "pylib";
    interp.exec("import sys");
    //interp.set("mis", mis);
    interp.exec("sys.path.append('" + path + "')");
    //interp.exec("import mipylib");
    //interp.exec("from mipylib.miscript import *");
    //interp.exec("from meteoinfo.numeric.JNumeric import *");
    for (String jarfn : jarfns) {
        interp.exec("sys.path.append('" + jarfn + "')");
    }
    interp.execfile(fn);
    System.exit(0);
}
 

/** Create executor for jython scripts
 *  @param support {@link ScriptSupport}
 */
public JythonScriptSupport(final ScriptSupport support) throws Exception
{
    this.support = support;

    // Concurrent creation of python interpreters has in past resulted in
    //     Lib/site.py", line 571, in <module> ..
    //     Lib/sysconfig.py", line 159, in _subst_vars AttributeError: {'userbase'}
    // or  Lib/site.py", line 122, in removeduppaths java.util.ConcurrentModificationException
    // Sync. on JythonScriptSupport to serialize the interpreter creation and avoid above errors.
    final long start = System.currentTimeMillis();
    synchronized (JythonScriptSupport.class)
    {
        // Could create a new 'state' for each interpreter
        // ++ Seems 'correct' since each interpreter then has its own path etc.
        // -- In scan server, some instances of the PythonInterpreter seemed
        //    to fall back to the default PySystemState even though
        //    a custom state was provided. Seemed related to thread local,
        //    not fully understood.
        // -- Using a new PySystemState adds about 3 second startup time,
        //    while using the default state only incurs that 3 second delay
        //    on very first access.
        // ==> Not using state = new PySystemState();
        final PySystemState state = null;
        python = new PythonInterpreter(null, state);
    }
    final long end = System.currentTimeMillis();
    logger.log(Level.FINE, "Time to create jython: {0} ms", (end - start));
}
 

/**
 * 获取python系统状态,可根据需要指定classloader/sys.stdin/sys.stdout等
 * 
 * @return PySystemState
 */
private PySystemState getPySystemState() {
	Properties props = new Properties();
	props.put("python.console.encoding", "UTF-8");
	props.put("sys.stdout.encoding", "UTF-8");
	props.put("python.security.respectJavaAccessibility", "false");
	props.put("python.import.site", "false");
	Properties preprops = System.getProperties();

	PySystemState.initialize(props, preprops);
	final PySystemState py = new PySystemState();
	py.setClassLoader(getClass().getClassLoader());
	return py;
}
 

/**
 * Prints a welcome message to the console.
 */
private void welcome() {
	console.getOutWriter().println("Python Interpreter for Ghidra");
	console.getOutWriter().println("Based on Jython version " + PySystemState.version);
	console.getOutWriter().println("Press 'F1' for usage instructions");
}
 

ThreadSafePythonInterpreter(PySystemState systemState) {
    super(null, systemState, true);
}
 

private static void runScript(String args[], String fn, int idx) {
    //String ext = GlobalUtil.getFileExtension(fn);
    //registerFonts();
    org.meteoinfo.global.util.FontUtil.registerWeatherFont();

    System.out.println("Running Jython script...");
    PySystemState state = new PySystemState();
    if (args.length > idx + 1) {
        for (int i = idx + 1; i < args.length; i++) {
            state.argv.append(Py.newStringOrUnicode(args[i]));
            //state.argv.append(new PyString(args[i]));
        }
    }
    //state.setdefaultencoding("utf-8");
    //PythonInterpreter interp = new PythonInterpreter(null, state);
    MyPythonInterpreter interp = new MyPythonInterpreter(null, state);

    boolean isDebug = java.lang.management.ManagementFactory.getRuntimeMXBean().
            getInputArguments().toString().contains("jdwp");
    String path, toolboxPath, miPath;
    if (isDebug) {
        path = "D:/MyProgram/java/MeteoInfoDev/MeteoInfo/MeteoInfoLab/pylib";
        toolboxPath = "D:/MyProgram/java/MeteoInfoDev/toolbox";
        miPath = "D:/MyProgram/Distribution/Java/MeteoInfo/MeteoInfo";
    } else {
        //String pluginPath = GlobalUtil.getAppPath(FrmMain.class) + File.separator + "plugins";
        //List<String> jarfns = GlobalUtil.getFiles(pluginPath, ".jar");
        miPath = GlobalUtil.getAppPath(FrmMain.class);
        String os = System.getProperty("os.name").toLowerCase();
        if (os.contains("windows") && miPath.substring(0, 1).equals("/")) {
            miPath = miPath.substring(1);
        }
        path = miPath + File.separator + "pylib";
        toolboxPath = miPath + "/toolbox";;
    }

    try {
        interp.exec("import sys");
        interp.exec("import os");
        interp.exec("import datetime");
        interp.exec("sys.path.append('" + path + "')");
        //interp.exec("from milab import *");
        interp.execfile(path + "/milab.py");
        if (!isDebug) {
            interp.exec("sys.path.append('" + toolboxPath + "')");
            //interp.exec("from toolbox import *");
        }
        interp.exec("mipylib.plotlib.miplot.batchmode = True");
        interp.exec("mipylib.plotlib.miplot.isinteractive = False");
        interp.exec("mipylib.migl.mifolder = '" + miPath + "'");
        System.out.println("mipylib is loaded...");
        interp.execfile(fn);
    } catch (Exception e) {
        e.printStackTrace();
        //System.exit(0);
    } finally {
        System.exit(0);
    }
}
 

/** Perform static, one-time initialization */
private static boolean init()
{
    final List<String> paths = new ArrayList<>();
    try
    {
        final Properties pre_props = System.getProperties();
        final Properties props = new Properties();

        // Compare jython setup with
        // org.csstudio.display.builder.runtime.script.internal.JythonScriptSupport

        // Disable cachedir to avoid creation of cachedir folder.
        // See http://www.jython.org/jythonbook/en/1.0/ModulesPackages.html#java-package-scanning
        // and http://wiki.python.org/jython/PackageScanning
        props.setProperty(RegistryKey.PYTHON_CACHEDIR_SKIP, "true");

        // By default, Jython compiler creates bytecode files xxx$py.class
        // adjacent to the *.py source file.
        // They are owned by the current user, which typically results in
        // problems for other users, who can either not read them, or not
        // write updates after *.py changes.
        Options.dont_write_bytecode = true;

        // With python.home defined, there is no more
        // "ImportError: Cannot import site module and its dependencies: No module named site"
        // Skipping the site import still results in faster startup
        props.setProperty("python.import.site", "false");

        // Prevent: console: Failed to install '': java.nio.charset.UnsupportedCharsetException: cp0.
        props.setProperty("python.console.encoding", "UTF-8");

        // Options: error, warning, message (default), comment, debug
        // props.setProperty("python.verbose", "debug");

        // No need to add numjy,
        // it's found on __pyclasspath__ because it's in the scan-model module

        // Add scan script paths
        for (String pref_path : ScanServerInstance.getScanConfig().getScriptPaths())
        {
            // Resolve built-in examples
            if (pref_path.startsWith(PathStreamTool.EXAMPLES))
            {
                String path = PathStreamTool.patchExamplePath(pref_path);
                final URL url = ScanServerInstance.class.getResource(path);
                if (url == null)
                    throw new Exception("Error in scan script path " + pref_path);
                path = url.toExternalForm();
                // Patch file:/path/to/the_file.jar!/path/within
                if (path.startsWith("file:"))
                    path = path.substring(5);
                path = path.replace(".jar!", ".jar");
                paths.add(path);
            }
            else // Add as-is
                paths.add(pref_path);
        }

        props.setProperty("python.path", paths.stream().collect(Collectors.joining(java.io.File.pathSeparator)));

        PythonInterpreter.initialize(pre_props, props, new String[0]);
        final PySystemState state = Py.getSystemState();
        final PyVersionInfo version = PySystemState.version_info;
        logger.log(Level.INFO, "Initial Paths for Jython " + version.major + "." + version.minor + "." + version.micro + ":");
        for (Object o : state.path)
            logger.log(Level.INFO, " * " + Objects.toString(o));

        // Display Builder Scripts would sometimes fail in "from ... import ..." with this error:
        //
        // File "..jython-standalone-2.7.1.jar/Lib/warnings.py", line 226, in warn
        // IndexError: index out of range: 0
        //
        // That version of Lib/warnings.py:226 tries to read sys.argv[0],
        // so setting sys.argv[0] avoids the crash.
        state.argv.clear();
        state.argv.add("ScanServerScript");
    }
    catch (Exception ex)
    {
        logger.log(Level.SEVERE, "Once this worked OK, but now the Jython initialization failed. Don't you hate computers?", ex);
        return false;
    }
    return true;
}
 

/** Perform static, one-time initialization */
private static boolean init()
{
    try
    {
        final Properties pre_props = System.getProperties();
        final Properties props = new Properties();

        // Jython 2.7(b3) needed these to set sys.prefix and sys.executable.
        // Locate the jython plugin for 'home' to allow use of /Lib in there
        // final String home = null; // getPluginPath("org.python.jython", "/");

        // If left undefined, initialization of Lib/site.py fails with
        // posixpath.py", line 394, in normpath AttributeError:
        // 'NoneType' object has no attribute 'startswith'
        // props.setProperty("python.home", home);
        // props.setProperty("python.executable", "None");

        // Disable cachedir to avoid creation of cachedir folder.
        // See http://www.jython.org/jythonbook/en/1.0/ModulesPackages.html#java-package-scanning
        // and http://wiki.python.org/jython/PackageScanning
        props.setProperty(RegistryKey.PYTHON_CACHEDIR_SKIP, "true");

        // By default, Jython compiler creates bytecode files xxx$py.class
        // adjacent to the *.py source file.
        // They are owned by the current user, which typically results in
        // problems for other users, who can either not read them, or not
        // write updates after *.py changes.
        // There is no way to have them be created in a different, per-user directory.
        // C Python honors an environment variable PYTHONDONTWRITEBYTECODE=true to
        // disable its bytecode files, but Jython only checks that in its command line launcher.
        // Use the same environment variable in case it's defined,
        // and default to disabled bytecode, i.e. the safe alternative.
        if (System.getenv("PYTHONDONTWRITEBYTECODE") == null)
            Options.dont_write_bytecode = true;
        else
            Options.dont_write_bytecode = Boolean.parseBoolean(System.getenv("PYTHONDONTWRITEBYTECODE"));

        // With python.home defined, there is no more
        // "ImportError: Cannot import site module and its dependencies: No module named site"
        // Skipping the site import still results in faster startup
        props.setProperty("python.import.site", "false");

        // Prevent: console: Failed to install '': java.nio.charset.UnsupportedCharsetException: cp0.
        props.setProperty("python.console.encoding", "UTF-8");

        // This will replace entries found on JYTHONPATH
        final String python_path = Preferences.python_path;
        if (! python_path.isEmpty())
            props.setProperty("python.path", python_path);

        // Options: error, warning, message (default), comment, debug
        // props.setProperty("python.verbose", "debug");
        // org.python.core.Options.verbose = Py.DEBUG;

        PythonInterpreter.initialize(pre_props, props, new String[0]);
        final PySystemState state = Py.getSystemState();
        final PyList paths = state.path;

        // Add the examples/connect2j to path.
        // During development, examples are in
        // "file:/some/path/phoebus/applications/display/model/target/classes/examples"
        final String examples = ModelPlugin.class.getResource("/examples").toString();
        if (examples.startsWith("file:"))
            paths.add(examples.substring(5) + "/connect2j");
        // In the compiled version, examples are in
        // "jar:file:/some/path/display-model-0.0.1.jar!/examples"
        else if (examples.startsWith("jar:file:"))
            paths.add(examples.substring(9).replace("jar!/", "jar/") + "/connect2j");
        else
            logger.log(Level.WARNING, "Cannot locate examples/connect2j from " + examples);

        final PyVersionInfo version = PySystemState.version_info;
        logger.log(Level.INFO, "Initial Paths for Jython " + version.major + "." + version.minor + "." + version.micro + ": " + paths);

        // Scripts would sometimes fail in "from ... import ..." with this error:
        //
        // File "..jython-standalone-2.7.1.jar/Lib/warnings.py", line 226, in warn
        // IndexError: index out of range: 0
        //
        // That version of Lib/warnings.py:226 tries to read sys.argv[0],
        // so setting sys.argv[0] avoids the crash.
        // Since state is shared by all scripts in a display,
        // set it to a generic "DisplayBuilderScript"
        state.argv.clear();
        state.argv.add("DisplayBuilderScript");

        return true;
    }
    catch (Exception ex)
    {
        logger.log(Level.SEVERE, "Once this worked OK, but now the Jython initialization failed. Don't you hate computers?", ex);
    }
    return false;
}
 

public static void addPathToPySystemState(PySystemState sys, String path) {
	try {
		sys.path.append(new PyString(path));
	} catch (Exception e){}
}
 

private void resetInterpreter() {
    interp = new PythonInterpreter(null, new PySystemState());
    sys = Py.getSystemState();
}
 

/**
 * Constructor
 * @param obj PyObject
 * @param state State
 */
public MyPythonInterpreter(PyObject obj, PySystemState state){
    super(obj, state);
    this.cflags.source_is_utf8 = true;
}