Skip to content

Commit

Permalink
(improvements) improve graalvm polyglot implementation to make it mor…
Browse files Browse the repository at this point in the history 
…e robust and reliable, especially when it comes to error states in scripts, supporting installed graalpy packages ( like pandas, numpy, ...)
  • Loading branch information
@ake2l
ake2l committed Jan 22, 2024
1 parent 623ac23 commit bd74a53
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 107 deletions.
118 changes: 69 additions & 49 deletions src/main/java/com/rapiddweller/benerator/script/GraalValueConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

package com.rapiddweller.benerator.script;

import com.rapiddweller.common.ConversionException;
Expand All @@ -32,76 +31,97 @@
import com.rapiddweller.model.data.Entity;
import org.graalvm.polyglot.Value;

import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;

/**
* Convert Graal Values into Java Types
* https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Value.html
* <p>
* Created at 30.12.2020
*
* @author Alexander Kell
* @since 1.1.0
* Converts GraalVM Polyglot Values into corresponding Java types.
* Throws RuntimeException if a value exceeds the int limits.
*/
public class GraalValueConverter extends ThreadSafeConverter<Value, Object> {

/**
* Instantiates a new Graal value converter.
*/
private static final Logger logger = Logger.getLogger(GraalValueConverter.class.getName());

public GraalValueConverter() {
super(Value.class, Object.class);
}

/**
* Value 2 java converter object.
*
* @param value the value
* @return the object
*/
public static Object value2JavaConverter(Value value) {
if (value.fitsInInt()) {
return value.asInt();
} else if (value.hasArrayElements()) {
return getArrayFromValue(value);
} else if (value.fitsInLong()) {
return value.asLong();
} else if (value.fitsInFloat()) {
return value.asFloat();
} else if (value.fitsInByte()) {
return value.asByte();
} else if (value.fitsInDouble()) {
return value.asDouble();
} else if (value.isString()) {
return value.asString();
} else if (value.isHostObject()) {
return value.asHostObject();
} else if (value.isBoolean()) {
return value.asBoolean();
} else if (value.isDate()) {
return value.asDate();
} else if (value.isNativePointer()) {
return value.asNativePointer();
} else if (value.hasMembers()) {
// Convert the value to a java.util.Map
Entity result = new Entity((ComplexTypeDescriptor) null);
value.getMemberKeys().forEach(key -> result.setComponent(key, value2JavaConverter(value.getMember(key))));
return result;
} else {
return null;
return value2JavaConverter(value, new HashMap<>(), 0);
}

private static Object value2JavaConverter(Value value, Map<Value, Object> referenceMap, int depth) throws ConversionException {
logger.fine("Converting value at depth " + depth + ": " + value);
if (referenceMap.containsKey(value)) {
logger.fine("Value already processed, returning cached result.");
return referenceMap.get(value);
}

Object result;
try {
if (value.fitsInInt()) {
result = value.asInt();
} else if (value.fitsInLong()) {
result = handleLongValue(value);
} else if (value.hasArrayElements()) {
result = getArrayFromValue(value, referenceMap, depth);
} else if (value.isString()) {
result = value.asString();
} else if (value.isHostObject()) {
return value.asHostObject();
} else if (value.isBoolean()) {
return value.asBoolean();
} else if (value.isDate()) {
return value.asDate();
} else if (value.isNativePointer()) {
return value.asNativePointer();
} else if (value.hasMembers()) {
result = convertValueToEntity(value, referenceMap, depth);
} else {
result = null;
}

referenceMap.put(value, result);
} catch (Exception e) {
logger.severe("Error converting value: " + e.getMessage());
throw new ConversionException("Error converting value", e);
}

return result;
}

private static Object getArrayFromValue(Value val) {
private static Object handleLongValue(Value value) {
long longValue = value.asLong();
if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) {
throw new RuntimeException("Value exceeds int limits: " + longValue);
}
return (int) longValue;
}

private static Object[] getArrayFromValue(Value val, Map<Value, Object> referenceMap, int depth) throws ConversionException {
Object[] out = new Object[(int) val.getArraySize()];
for (int i = 0; i < val.getArraySize(); i++) {
out[i] = value2JavaConverter(val.getArrayElement(i));
out[i] = value2JavaConverter(val.getArrayElement(i), referenceMap, depth + 1);
}

return out;
}

private static Entity convertValueToEntity(Value value, Map<Value, Object> referenceMap, int depth) {
Entity result = new Entity((ComplexTypeDescriptor) null);
value.getMemberKeys().forEach(key -> {
try {
result.setComponent(key, value2JavaConverter(value.getMember(key), referenceMap, depth + 1));
} catch (ConversionException e) {
logger.warning("Error converting entity member: " + key + "; " + e.getMessage());
}
});
return result;
}

@Override
public Object convert(Value sourceValue) throws ConversionException {
return value2JavaConverter(sourceValue);
}
}

141 changes: 83 additions & 58 deletions src/main/java/com/rapiddweller/benerator/script/PolyglotContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,79 +9,104 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.nio.file.Paths;
import java.util.Map;
import java.util.Objects;

class PolyglotContext {

private static final Logger logger = LoggerFactory.getLogger(PolyglotContext.class);
private String previousMissingObject;
private final org.graalvm.polyglot.Context polyglotCtx;
private static final Logger logger = LoggerFactory.getLogger(PolyglotContext.class);
private String previousMissingObject;
private final org.graalvm.polyglot.Context polyglotCtx;

public void updatePolyglotLocalFromGlobal(Context context, String language) {
for (String entry : polyglotCtx.getBindings(language).getMemberKeys()) {
migrateBeneratorContext2GraalVM(context, language, entry);
}
public void updatePolyglotLocalFromGlobal(Context context, String language) {
for (String entry : polyglotCtx.getBindings(language).getMemberKeys()) {
migrateBeneratorContext2GraalVM(context, language, entry);
}
}

public void migrateBeneratorContext2GraalVM(Context context, String language, String valueKey) {
Object valueType;
try {
Object obj = context.get(valueKey);
if (obj != null) {
valueType = obj.getClass();
// check if Entity Object
if (Entity.class.equals(valueType)) {
logger.debug("Entity found : {}", valueKey);
Map<String, Object> map = new Entity2MapConverter().convert((Entity) obj);
// to access items of map in polyglotCtx it is nessesary to create an ProxyObject
ProxyObject proxy = ProxyObject.fromMap(map);
polyglotCtx.getBindings(language).putMember(valueKey, proxy);
} else {
logger.debug("{} found : {}", valueType.getClass(), valueKey);
polyglotCtx.getBindings(language).putMember(valueKey, obj);
}
}
} catch (NullPointerException e) {
logger.error("Context {} was NULL, this should not happen!", context);
public void migrateBeneratorContext2GraalVM(Context context, String language, String valueKey) {
Object valueType;
try {
Object obj = context.get(valueKey);
if (obj != null) {
valueType = obj.getClass();
// check if Entity Object
if (Entity.class.equals(valueType)) {
logger.debug("Entity found : {}", valueKey);
Map<String, Object> map = new Entity2MapConverter().convert((Entity) obj);
// to access items of map in polyglotCtx it is nessesary to create an ProxyObject
ProxyObject proxy = ProxyObject.fromMap(map);
polyglotCtx.getBindings(language).putMember(valueKey, proxy);
} else {
logger.debug("{} found : {}", valueType.getClass(), valueKey);
polyglotCtx.getBindings(language).putMember(valueKey, obj);
}

}
} catch (NullPointerException e) {
logger.error("Context {} was NULL, this should not happen!", context);
}

synchronized public Value evalScript(Context context, String text, String language) throws ScriptException {
}

Value returnValue = null;
try {
this.updatePolyglotLocalFromGlobal(context, language);
returnValue = polyglotCtx.eval(language, text);
} catch (org.graalvm.polyglot.PolyglotException e) {
if (e.getMessage().contains("ReferenceError: ")) {
String missingObject = e.getMessage().replace("ReferenceError: ", "").replace(" is not defined", "");
if (!Objects.equals(previousMissingObject, missingObject)) {
this.migrateBeneratorContext2GraalVM(context, language, missingObject);
returnValue = evalScript(context, text, language);
previousMissingObject = missingObject;
}
}
else if (e.getMessage().contains(("NameError: "))) {
String missingObject = e.getMessage().replace("NameError: name '", "").replace("' is not defined", "");
if (!Objects.equals(previousMissingObject, missingObject)) {
this.migrateBeneratorContext2GraalVM(context, language, missingObject);
returnValue = evalScript(context, text, language);
previousMissingObject = missingObject;
}
}
else {
throw new ScriptException(e.getMessage(), null);
}
synchronized public Value evalScript(Context context, String text, String language) throws ScriptException {
final int MAX_ATTEMPTS = 3; // Maximum attempts for the same missing object
Value returnValue = null;

try {
this.updatePolyglotLocalFromGlobal(context, language);
returnValue = polyglotCtx.eval(language, text);
} catch (org.graalvm.polyglot.PolyglotException e) {
String missingObject = extractMissingObjectName(e.getMessage());
if (missingObject != null) {
if (!Objects.equals(previousMissingObject, missingObject)) {
previousMissingObject = missingObject;
missingObjectAttempts = 1; // Reset the attempt count for a new missing object
this.migrateBeneratorContext2GraalVM(context, language, missingObject);
returnValue = evalScript(context, text, language);
} else if (missingObjectAttempts < MAX_ATTEMPTS) {
missingObjectAttempts++; // Increment attempt count for the same missing object
this.migrateBeneratorContext2GraalVM(context, language, missingObject);
returnValue = evalScript(context, text, language);
} else {
throw new ScriptException("This object couldn't be found: " + missingObject, null);
}
return returnValue;
} else {
throw new ScriptException(e.getMessage(), null);
}
}
return returnValue;
}

private String extractMissingObjectName(String errorMessage) {
if (errorMessage.contains("ReferenceError: ")) {
return errorMessage.replace("ReferenceError: ", "").replace(" is not defined", "");
} else if (errorMessage.contains("NameError: ")) {
return errorMessage.replace("NameError: name '", "").replace("' is not defined", "");
}
return null;
}

public PolyglotContext() {
this.polyglotCtx = org.graalvm.polyglot.Context
.newBuilder("js", "python")
.allowAllAccess(true).build();
private int missingObjectAttempts = 0;

public PolyglotContext() {

if (isGraalVM()) {
this.polyglotCtx = org.graalvm.polyglot.Context
.newBuilder("js", "python")
.allowIO(true)
.option("python.ForceImportSite", "true")
.allowAllAccess(true).build();
} else {
this.polyglotCtx = org.graalvm.polyglot.Context
.newBuilder("js")
.allowIO(true)
.allowAllAccess(true).build();
}
}

private boolean isGraalVM() {
String javaVmName = System.getProperty("org.graalvm.home");
return javaVmName != null;
}
}

0 comments on commit bd74a53

Please sign in to comment.