acidcam: input[/home/jared/Videos/freddy.mov] output[fredtest.mov] width[592] height[448] fps[29.92] length[37.03 seconds] format[MPEG-4 (Quicktime)]

Filters to Apply:
Plugin
acidcam: Working frame: [0/1108] - 0% Size: 0 MB

Thread 1 "acidcam" received signal SIGSEGV, Segmentation fault.
0x00007fffcd3a1bcf in PyErr_SetObject () from /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
(gdb) backtrace
#0 0x00007fffcd3a1bcf in PyErr_SetObject () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#1 0x00007fffcd3a224f in PyErr_SetString () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#2 0x00007fffcc51c93e in _wrap_metacall () at /usr/local/lib/_py_port.so
#3 0x00007fffcd4549eb in _PyCFunction_FastCallDict () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#4 0x00007fffcd3bce0c in () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#5 0x00007fffcd3c3092 in _PyEval_EvalFrameDefault () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#6 0x00007fffcd3bc63f in () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#7 0x00007fffcd3bd0fe in PyEval_EvalCodeEx () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#8 0x00007fffcd47d782 in () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#9 0x00007fffcd4aa2d8 in PyObject_Call () at /usr/lib/x86_64-linux-gnu/libpython3.6m.so.1.0
#10 0x00007fffcd90386e in function_py_interface_invoke () at /usr/local/lib/libpy_loader.so
#11 0x00007ffff65d398b in metacallt () at /usr/local/lib/libmetacall.so
#12 0x000055555556680e in plugin_callback(cv::Mat&) (frame=...) at acidcam-cli-main.cpp:138
#13 0x00007ffff784bf22 in ac::CallFilter(int, cv::Mat&) (index=, frame=...) at ac-filter1.cpp:146
#14 0x0000555555563eed in cmd::AC_Program::run() (this=0x55555576e720 ) at acidcam-cli.cpp:441
#15 0x000055555555f3dd in main(int, char**) (argc=9, argv=0x7fffffffdea8) at acidcam-cli-main.cpp:700
(gdb)

if I comment out val = metacall('matrix_getpixel',matrix, i, z) it does not crash.
if this is something on my side I apologize for bothering.

Python code:

import random, sys
sys.path.append("/usr/local/lib")
from metacall import metacall

def array_size(width: int, height: int) -> int:
global screen_width
global screen_height
global pitch_value
screen_width = width
screen_height = height
pitch_value = width*3
return 0

def filter(matrix):
for z in range(0, screen_height-1):
for i in range(0, screen_width-1):
val = metacall('matrix_getpixel',matrix, i, z)
metacall('matrix_setpixel',matrix, i, z, 0)
return 0

C++ code:

#if METACALL_ENABLED == 1

void init_metacall();
void *matrix_SetPixel(void *args[]);
void *matrix_GetPixel(void *args[]);

void init_metacall() {
metacall_register("matrix_setpixel", matrix_SetPixel, METACALL_PTR,4, METACALL_PTR, METACALL_INT, METACALL_INT, METACALL_INT);
metacall_register("matrix_getpixel", matrix_GetPixel, METACALL_ARRAY, 3, METACALL_PTR, METACALL_INT, METACALL_INT);

//metcall_register("matrix_setpixel_rgb", matrix_SetPixelRGB, METACALL_PTR, 7, METACALL_PTR, METACALL_INT, METACALL_INT, METACALL_INT, METACALL_INT, METACALL_INT)
//metacall_register("matrix_getpixel", matrix_GetPixel, METACALL_INT,3, METACALL_PTR, METACALL_INT, METACALL_INT);

}

void *matrix_SetPixel(void args[]) {
cv::Mat type = (cv::Mat)metacall_value_to_ptr(args[0]);
int x = metacall_value_to_int(args[1]);
int y = metacall_value_to_int(args[2]);
int color_value = metacall_value_to_int(args[3]);
const unsigned char bgr = (const unsigned char)&color_value;
type->atcv::Vec3b(y, x) = cv::Vec3b(bgr[0], bgr[1], bgr[2]);
return metacall_value_create_ptr((void)type);
}

void *matrix_GetPixel(void *args[]) {
cv::Mat type = (cv::Mat)metacall_value_to_ptr(args[0]);
int x = metacall_value_to_long(args[1]);
int y = metacall_value_to_long(args[2]);
cv::Vec3b color_v = type->atcv::Vec3b(y, x);
void *array[] = {metacall_value_create_char(color_v[0]), metacall_value_create_char(color_v[1]),metacall_value_create_char(color_v[2])};
return metacall_value_create_array((const void **)array, sizeof(array)/sizeof(array[0]));
}

#endif

For init stuff i have:

in main function:
#if METACALL_ENABLED == 1
metacall_log_stdio_type log_stdio = { stdout };
metacall_log(METACALL_LOG_STDIO, (void *)&log_stdio);
metacall_initialize();
#endif

in Program class

bool AC_Program::loadPlugin(const std::string &s) {
    if(s.find(".py") != std::string::npos) {

#if METACALL_ENABLED == 1
std::string dir = s;
std::string path = dir.substr(0, dir.rfind("/"));
std::string script_name=s.substr(s.rfind("/")+1, s.length());
std::cout << "Attempting to load: " << script_name << "\n";
const char * py_scripts[] = {
script_name.c_str(),
};
if (metacall_load_from_file("py", py_scripts, sizeof(py_scripts) / sizeof(py_scripts[0]), NULL) == 0) {
return true;
}
#else
std::cerr << "Error Python Not Enabled...\n";
exit(EXIT_FAILURE);
#endif
} else {
library = dlopen(s.c_str(), RTLD_LAZY);
if(library == NULL)
return false;

        void *addr = dlsym(library, "filter");
        const char *err = dlerror();
        if(err) {
            std::cerr << "Could not locate function: filter in " << s << "\n";
            return false;
        }
        plugin = reinterpret_cast<plugin_filter>(addr);
    }
    return false;
}

🚀 Feature

This is a plan for implementing the LLVM Loader for MetaCall. This is subject to many changes and it is open to suggestions from everyone.

Proposed Solution

As a first step, I wish to develop a POC with the basic working functionality of what is expected.

• Emit the LLVM IR from C/C++ files using clang. These can be emitted in .ll or .bc formats.
• Load the LLVM IR (.ll) file using the parseIRFile method of LLVM C++ API and store it in a module.
• The main functionality would be to be able to implement the discover method. For this, we can iterate through the module and use the getFunctionList function of the API to fetch all the functions. If it is a valid function declaration, record this function and iterate through each of the functions to get their respective parameter lists.
• Another main functionality would be to be able to call the functions dynamically. For this, we can simply make an ExecutionEngine and find the function by its name (we can do this as we've already got all the functions in the discover function). Then, we can simply execute it by specifying their parameters.

As the second step, I wish to map the POC to the actual loader according to MetaCall's design decisions and actually start implementing it for the core.

• Perform mapping of various methods to proper places in the MetaCall loader in the core.
• Start implementing the MetaCall LLVM loader as a whole. (IN PROGRESS)

Proof of Concept (POC)

My LLVM Loader POC can be found here.

This is the test that reproduces the bug:
https://github.com/metacall/core/blob/627454ecae552918d2dbeee16414c64ceade800a/source/tests/metacall_duplicated_symbols_test/source/metacall_duplicated_symbols_test.cpp

Delete Swig dependency as soon as possible. List of Loaders and Ports which can be unified:

• Ruby Loader
• C# Loader
• JS (V8) Loader

Also implement the gem / nuget packages. But this is another task.

iOS and Android supported can be added using the NodeJS-mobile library

Add docs.rs and documentation. Documentation examples: https://doc.rust-lang.org/stable/rust-by-example/meta/doc.html

Feedback of @In-line and @SnejUgal from t.me/rustdevs:

call eax:
Few suggestions.

1. Don't use str error types. Use structured errors.
2. Any should have From/Into implementations for common types, also possibly serde serialize/deserialize
3. Any is confusing with standard library Any, rename it, pls.
4. This seems plain wrong, rust strings are not null terminated

   core/source/ports/rs_port/src/lib.rs

   Line 116 in 7d56d8c

   let mut c_scripts : Vec<*const u8> = scripts.clone().into_iter().map(|x| x.as_bytes().as_ptr()).collect();

SnejUgal:
you can write a proc macro that inspects loaded files at compile time and generates all the boilerplate

core/source/ports/rs_port/src/lib.rs

Lines 82 to 102 in 7d56d8c

	impl From<c_short> for Any {
	fn from(val : c_short) -> Self { Any::Short(val) }
	}
	impl From<c_int> for Any {
	fn from(val : c_int) -> Self { Any::Int(val) }
	}
	impl From<c_long> for Any {
	fn from(val : c_long) -> Self { Any::Long(val) }
	}
	impl From<c_char> for Any {
	fn from(val : c_char) -> Self { Any::Char(val as u8 as char) }
	}
	impl From<bool> for Any {
	fn from(val : bool) -> Self { Any::Bool(val) }
	}
	impl From<c_float> for Any {
	fn from(val : c_float) -> Self { Any::Float(val) }
	}
	impl From<c_double> for Any {
	fn from(val : c_double) -> Self { Any::Double(val) }
	}

all this seems to depend that these C's int, long etc are always equivalent to Rust's i32, i64 etc, while this is not true

call eax:
Few more suggestions.

Library code can be improved further.

1. Use RAII pattern
2. Don't constrain use to specific owned types like String, use references or conversion traits AsRef, etc..
3. Use cargo fmt, cargo clippy
4. Add separate metacall-c crate for raw bindings.
5. Remove unnecessary type annotations, rust can pretty much infer majority of them

For good example of idiomatic ffi you can look at neon, it for example has serde serialization for serializing type to JS Values

You use a lot of imperative code with unnecessary allocations (for loops, vectors, etc..), in Rust you can use iterators with operations like map, filter, for_each, some parts can be reduced to one liners with iterators.

🐛 Bug Report

When running metacall-node-port-test with Node 15.x, the program hangs after the tests finish (note that the fail test fails on Node 15.x due to a change in the error message, so this test must be commented out to reproduce).

Expected Behavior

metacall-node-port-test should gracefully close after the tests finish.

Current Behavior

metacall-node-port-test hangs after the tests finish.

Possible Solution

Putting node_loader_impl_print_handles calls in node_loader_impl_destroy_prepare_cb and node_loader_impl_destroy_safe, @viferga suspects it has something to do with a signal handle that is created on Node 15.x but not on 10.x. Further testing showed that the signal was SIGWINCH, which - for unknown reasons - was created in spite of the terminal size remaining constant during execution.

Filtering out SIGWINCH in node_loader_impl_walk_async_handles_count causes the program to segfault after finishing the tests. AddressSanitizer does not pick up on anything, but running this commit with Valgrind gives this output.

Steps to Reproduce

1. Comment out the fail test in source/ports/node_port/test/index.js
2. Build metacall-node-port-test using Node 15.x
3. Run metacall-node-port-test

https://travis-ci.org/github/metacall/core/jobs/741857383#L408

CMake Error at /usr/share/cmake-3.13/Modules/Internal/CPack/CPackRPM.cmake:767 (message):
RPM package requires rpmbuild executable
Call Stack (most recent call first):
/usr/share/cmake-3.13/Modules/Internal/CPack/CPackRPM.cmake:1870 (cpack_rpm_generate_package)
CPack Error: Error while execution CPackRPM.cmake
CPack Error: Error while execution CPackRPM.cmake
CPack Error: Error while execution CPackRPM.cmake
CPack Error: Problem compressing the directory
CPack Error: Error when generating package: metacall
make[3]: *** [deploy/CMakeFiles/pack-metacall.dir/build.make:57: deploy/CMakeFiles/pack-metacall] Error 1
make[2]: *** [CMakeFiles/Makefile2:10203: deploy/CMakeFiles/pack-metacall.dir/all] Error 2
make[1]: *** [CMakeFiles/Makefile2:10242: deploy/CMakeFiles/pack.dir/rule] Error 2

Implement this technique: bed13df for the rest of NodeJS calls (including await) and drop support for NodeJS 8.

Expected Behavior

Pressing Ctrl-D exits from metacallcli

Actual Behavior

Pressing Ctrl-D causes metacallcli to infinitely print >

Would it be of use to list related prior art in multiple computer language programming? My M.S. thesis involved that - http://ftp.sunet.se/mirror/archive/ftp.sunet.se/pub/lang/compilers-list/Chameleon.ps.z

📚 Documentation

I would want to elaborate a little on the loaders section and add a section on LLVM loaders along with some diagrams. These are not of high priority as of now but just opening the issue so that I don't forget about it.

After testing with NodeJS Port it seems that Ruby loader does not work properly, it needs to be thread safe. There are some workarounds to make it run. All of the have been commented in rb_loader_impl.c file.

A new branch has been created for it: https://github.com/metacall/core/tree/hotfix/ruby-loader-thread-safe

It's unclear what directory I need to include to utilize this library from C/C++. In the build folder, there's a source folder with many subfolders with include directories. Do I need to include all of those? Just for context, I'm working in Xcode trying to build this for iOS/macOS.

Side note if I can get this library to work for what I need I'd really love to donate to you. This library is godsend and would widen my options for other 3rd party libraries in my project.

When building solution in release mode I get the following errors:

There is a need to simplify install scripts or document them in the python port readme. The ideal would be to integrate metacall-{install, update, uninstall} with pip {install, uninstall}, but not sure if this is properly supported by pip or setuptools.

Reference: #8
Depends on:

• MetaCall cross-platform install script (https://github.com/metacall/install/blob/master/metacall-install.sh).
• Travis issues (#14, #13).

The idea is to simplify the API of MetaCall in order to make it natural for high level languages. So if we have a NodeJS module like:
asd.js

function hello() {
  console.log("hello world");
}
module.exports = {
  hello
};

We should be able to import it from Python like this (if the script is being executed by metacallcli):

import asd

asd.hello();

Or like this (if the script is being executed by python interpreter):

import metacall # Monkey Patch import
import asd

asd.hello();

To achieve this, we must implement the monkey patch mechanism into Python Port, and force all loaders to insert their respective ports into the loaded scripts. Forcing the loaders to import the ports at the beginning is an extra for removing the need of having metacall dependency in the requirements.txt (or equivalent).

The priority of this issue is to improve Python Port API with syntax sugar and make it transparent.

Here is a tutorial for hooking import in python, the tool we should use to implement the feature: https://pymotw.com/3/sys/imports.html .

The import class should search for the scripts in the folder obtained by the environment variable LOADER_SCRIPTS_PATH.

There is a problem related to python, it is impossible to specify the extension of the module to be loaded, so the ideal would be to have a mapping from file extensions to loaders:

{
  [js, node]: node, // this one should be used instead of js loader (v8) by default if both are loaded
  [py]: py,
  [vb, cs]: cs,
  [js]: js,
  [rb]: rb,
}

And the import mechanism should search for each file + extension, and if it exists, try to load it with the loader it belongs to.

The mechanism should be implemented here: https://github.com/metacall/core/blob/develop/source/ports/py_port/package/metacall/__init__.py

I am currently reading through the code for the Python loader and have found a few recurring patterns that I believe could be improved:

1. A lot of the error handling cleanup code is quite repetitive and the current structure is quite error-prone. This could be improved by using goto to localize the cleanup code (see this link for an example).
2. Variables are often declared at the start of scopes (i.e. C89 style), rather than when they are actually initialized. This makes it harder to reason about their scope and could be improved by only declaring the variables when they are to be initialized, thus making their scope as tight as possible.

If you agree with these observations, I will start working on implementing them in the codebase. Please see this commit for an example of how these changes might be implemented.

Travis CI is shutting down so we should update this https://github.com/metacall/core/blob/develop/.travis.yml to here:
https://github.com/metacall/core/blob/develop/.gitlab-ci.yml

📚 Documentation

The types Future, Function, Class, Object are not mentioned in the documentation of Type System under Reflect.

These are present in the codebase (under reflect module)

Is there any specific reason for this? I can add them if needed :)

This issue is to list out things that may or may not have been completed in order to get rs-port working.

• Add the cargo based rs-port project. // TODO: Will PR it when I can get cmake build working
• Use cmakeRust(might need some modifications) for the cmake build system integration.
• Setup cmake for the Rust project.

At the moment I haven't pushed the code because I haven't been able to get cmake working with any of it, but I just uploaded my source on metacall-rs-port.

cc: @viferga

Haskell has a compiler generation tool that provides the stubs in C language to Haskell (https://wiki.haskell.org/Calling_Haskell_from_C). Due to this design, the loader must be implemented mostly with Haskell. The load from file, memory, introspection, the FFI dynamic calls, and others must be implemented in Haskell side.

For conversion types, Haskell must return a structure containing the raw pointer plus information about how to retrieve the data stored in its custom types. This is achieved by Foreing.Storable (http://hackage.haskell.org/package/base-4.12.0.0/docs/Foreign-Storable.html) with the following C API (https://gitlab.haskell.org/ghc/ghc/blob/master/includes/rts/storage/InfoTables.h). The conversion can be done from C side meanwhile the dynamic call exposes its types with this interface.

Example of interfacing Haskell with Rust using Storage API: https://github.com/AndyShiue/ende/blob/155b772b9ae584deb987a4dc171f6cf0fa94e713/backend/src/trans.rs

In order to get compiler flags for GHC for C/C++ (or Cargo): https://github.com/AndyShiue/ende/blob/155b772b9ae584deb987a4dc171f6cf0fa94e713/frontend/get_args.sh

This hints and implementation ideas are thanks to @petercommand .

Other references:
Information about Storage and GC: https://stackoverflow.com/a/57154026
Threading and FFI: http://www.cs.toronto.edu/~trebla/personal/haskell/ghc-conc-ffi.xhtml
Alternative way of serialization of data types: https://github.com/nh2/call-haskell-from-anything

It would be helpful to have a function such as metacall_await_future in the C API for awaiting future values.

This can lift the need for introspecting loaded functions to know if they're async. metacallv_s (or other calling functions) could return a future value (with metacall_value_id 12) which could then be converted to the calling language's future type using metacall_await_future. No additional abstractions would be needed in order to handle async functions in MetaCall ports.

🐛 Bug Report

This bug (#65) has been solved, creating a breaking change in the C API semantics which led to v0.4.0. But still there is problems when loading duplicated symbols in different loaders in their global scope.

Here's the TODO:

Expected Behavior

This should give an error when trying to load:
a.py:

def a():
  return 9

b.js:

module.exports = { a: () => 3 };

main.c:

metacall_load_from_file("py", a_script_list, sizeof(a_script_list) / sizeof(a_script_list[0]), NULL); // 0
metacall_load_from_file("node", b_script_list, sizeof(b_script_list) / sizeof(b_script_list[0]), NULL); // 1

Current Behavior

In the current behavior (not tested), it will load both, and when calling the a function...:

metacall("a");

It will execute the first in the loaders list.

Possible Solution

The solution is to have a global context for all loaders, and pass all functions into this context, which we can pass by arguments to loader_impl module from loader module.

Recently @trgwii suggested the following addition for metacall.json:

{
  ...
  "imports": {
    "ramda": "https://raw.githubusercontent.com/ramda/ramda/v0.27.1/dist/ramda.min.js"
  }
}

And then do in Python (or any other language):

import ramda

An extension to the previous mechanism may be the following:

{
  "imports": {
    "ramda": "npm:ramda",
    "telethon": "pip3:telethon"
  }
}

Python is the language that would need import maps in order to implement this kind of imports. Other languages like Ruby or NodeJS may allow to do this easily like:

require 'https://raw.githubusercontent.com/ramda/ramda/v0.27.1/dist/ramda.min.js'

const R = require('https://raw.githubusercontent.com/ramda/ramda/v0.27.1/dist/ramda.min.js');

Rust interface is marked as safe, while underlying core implementation isn't thread safe.

This is unsound.

When running a NodeJS script with NodeJS (compiled with OpenSSL statically) and then loading any library in Python that uses Py__Hash (which is a C extension for Python using OpenSSL dynamically linked) it generates a segmentation fault (this problem can be reproduced by running this example with NodeJS: https://github.com/metacall/pdf-generator-email-sender-landing-page-example).

In this example Py__Hash is using OpenSSL 1.1. and NodeJS is using 1.0.2p:

{ http_parser: '2.8.0',
  node: '8.12.0',
  v8: '6.2.414.66',
  uv: '1.19.2',
  zlib: '1.2.11',
  ares: '1.10.1-DEV',
  modules: '57',
  nghttp2: '1.32.0',
  napi: '3',
  openssl: '1.0.2p',
  icu: '60.1',
  unicode: '10.0',
  cldr: '32.0',
  tz: '2017c' }

This is a fragment of the segmentation fault obtained with valgrind referring to OpenSSL initialization of Py_Hash.

==3013== Invalid read of size 1
==3013== at 0x5D8D520: __strcmp_sse2_unaligned (strcmp-sse2-unaligned.S:24)
==3013== by 0x123A409: lh_insert (in /usr/bin/node)
==3013== by 0x12458F0: OBJ_NAME_add (in /usr/bin/node)
==3013== by 0x1230FF5: EVP_add_digest (in /usr/bin/node)
==3013== by 0xC1034FA: ??? (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0xC11AA78: ??? (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0x5AEF758: __pthread_once_slow (pthread_once.c:116)
==3013== by 0xC171558: CRYPTO_THREAD_run_once (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0xC11AFB2: OPENSSL_init_crypto (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0xBB40B10: PyInit__hashlib (in /usr/lib/python3.5/lib-dynload/_hashlib.cpython-35m-x86_64-linux-gnu.so)
==3013== by 0xA30F70F: _PyImport_LoadDynamicModuleWithSpec (in /usr/lib/x86_64-linux-gnu/libpython3.5m.so.1.0)
==3013== by 0xA312FE6: ??? (in /usr/lib/x86_64-linux-gnu/libpython3.5m.so.1.0)
==3013== Address 0x0 is not stack'd, malloc'd or (recently) free'd
==3013==
==3013==
==3013== Process terminating with default action of signal 11 (SIGSEGV)
==3013== Access not within mapped region at address 0x0
==3013== at 0x5D8D520: __strcmp_sse2_unaligned (strcmp-sse2-unaligned.S:24)
==3013== by 0x123A409: lh_insert (in /usr/bin/node)
==3013== by 0x12458F0: OBJ_NAME_add (in /usr/bin/node)
==3013== by 0x1230FF5: EVP_add_digest (in /usr/bin/node)
==3013== by 0xC1034FA: ??? (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0xC11AA78: ??? (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0x5AEF758: __pthread_once_slow (pthread_once.c:116)
==3013== by 0xC171558: CRYPTO_THREAD_run_once (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0xC11AFB2: OPENSSL_init_crypto (in /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1)
==3013== by 0xBB40B10: PyInit__hashlib (in /usr/lib/python3.5/lib-dynload/_hashlib.cpython-35m-x86_64-linux-gnu.so)
==3013== by 0xA30F70F: _PyImport_LoadDynamicModuleWithSpec (in /usr/lib/x86_64-linux-gnu/libpython3.5m.so.1.0)
==3013== by 0xA312FE6: ??? (in /usr/lib/x86_64-linux-gnu/libpython3.5m.so.1.0)
==3013== If you believe this happened as a result of a stack
==3013== overflow in your program's main thread (unlikely but
==3013== possible), you can try to increase the size of the
==3013== main thread stack using the --main-stacksize= flag.
==3013== The main thread stack size used in this run was 8388608.

For Netcore 1.x (and maybe 2.x) it is possible to properly handle dependencies (not sure if it is involved with the current fail when loading.

Current error:
The type 'Console' exists in both 'System.Console, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a' and 'System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e'

...
"Microsoft.Extensions.DependencyModel": "2.1.0",
...

using Microsoft.Extensions.DependencyModel;
using Microsoft.Extensions.DependencyModel.Resolution;

        internal sealed class AssemblyResolver : IDisposable
        {
            private ICompilationAssemblyResolver assemblyResolver;
            private DependencyContext dependencyContext;
            private AssemblyLoadContext loadContext;

            public AssemblyResolver(MemoryStream stream)
            {
                this.Assembly = AssemblyLoadContext.Default.LoadFromStream(stream);
                this.Dependency(Environment.GetEnvironmentVariable("LOADER_SCRIPT_PATH"));
            }

            public AssemblyResolver(AssemblyName assemblyName)
            {
                this.Assembly = AssemblyLoadContext.Default.LoadFromAssemblyName(assemblyName);
                this.Dependency(Environment.GetEnvironmentVariable("LOADER_SCRIPT_PATH"));
            }

            public AssemblyResolver(string path)
            {
                this.Assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
                this.Dependency(path);
            }

            public Assembly Assembly { get; }

            public void Dispose()
            {
                this.loadContext.Resolving -= this.OnResolving;
            }

            private void Dependency(string path)
            {
                this.dependencyContext = DependencyContext.Load(this.Assembly);

                this.assemblyResolver = new CompositeCompilationAssemblyResolver(new ICompilationAssemblyResolver[]
                {
                    new AppBaseCompilationAssemblyResolver(Path.GetDirectoryName(path)),
                    new ReferenceAssemblyPathResolver(),
                    new PackageCompilationAssemblyResolver()
                });

                this.loadContext = AssemblyLoadContext.GetLoadContext(this.Assembly);
                this.loadContext.Resolving += OnResolving;
            }

            private Assembly OnResolving(AssemblyLoadContext context, AssemblyName name)
            {
                RuntimeLibrary library = this.dependencyContext.RuntimeLibraries.FirstOrDefault(runtime =>
                    string.Equals(runtime.Name, name.Name, StringComparison.OrdinalIgnoreCase));

                if (library != null)
                {
                    var wrapper = new CompilationLibrary(
                        library.Type,
                        library.Name,
                        library.Version,
                        library.Hash,
                        library.RuntimeAssemblyGroups.SelectMany(g => g.AssetPaths),
                        library.Dependencies,
                        library.Serviceable);

                    var assemblies = new List<string>();
                    this.assemblyResolver.TryResolveAssemblyPaths(wrapper, assemblies);
                    if (assemblies.Count > 0)
                    {
                        return this.loadContext.LoadFromAssemblyPath(assemblies[0]);
                    }
                }

                return null;
            }
        }

https://stackoverflow.com/a/51115745

Description

As discussed with @josead we would be adding some Issues + PR templates for making the Contribution Workflow better.

cc: @viferga

🚀 Feature

See the ideas page for more information.

To Do
The following checklists will become more specific in future
[ ] import llvm libraries
[ ] initialize
[ ] introspect

Related to this commit: 3fb57a8#diff-fe9caa8952071583fbfe3ea54361f8843196e62d4eeb269125b5f70494caca2fL842

I have avoided a segmentation fault when executing metacall_destroy in the Node Port test. This should be reviewed. Probably it is a double free or something like that. I prefer to leave it as a memory leak on destroy rather than a segmentation fault.

It would be pretty awesome to have docker as a backend with exported Bash commands that gets pwd as variable and mount some core paths to guest to persist even when a new version is released.

I.g

metacall pip3 install x

Pip folder with installed modules should be available on guest in a path such as

/opt/metacall/pip

and mounted via Docker run to the container.

It would be very useful if there's a function in the C API that returns whether a loaded function is async or not.

Reference: #8 (comment)
Needs to be solved first: #13

The publish must be done only when tagging. Environment variables with secrets must be tested and enabled in the upload script.

When trying to build the C# Loader, this happens:

/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : Unable to load the service index for source https://api.nuget.org/v3/index.json. [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : The SSL connection could not be established, see inner exception. [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : Authentication failed, see inner exception. [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : The type initializer for 'SslMethods' threw an exception. [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : The type initializer for 'Ssl' threw an exception. [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : The type initializer for 'SslInitializer' threw an exception. [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
/usr/share/dotnet/sdk/2.2.402/NuGet.targets(123,5): error : error:0E076071:configuration file routines:MODULE_RUN:unknown module name [/usr/local/metacall/source/loaders/cs_loader/netcore/source/project.csproj]
�[91mmake[2]: *** [source/loaders/cs_loader/netcore/CMakeFiles/cs_loader_impl.dir/build.make:57: source/loaders/cs_loader/netcore/CMakeFiles/cs_loader_impl] Error 1
make[2]: Target 'source/loaders/cs_loader/netcore/CMakeFiles/cs_loader_impl.dir/build' not remade because of errors.
�[0m�[91mmake[1]: *** [CMakeFiles/Makefile2:6668: source/loaders/cs_loader/netcore/CMakeFiles/cs_loader_impl.dir/all] Error 2

This has happened before to me. It basically happens when you lose the connection to Internet, but I do not discard it can happen also because of some lack of certificates, or whatever random reason.

All tests with C# Loader involved fail. This results into:

The following tests FAILED:
19 - cs-loader-test (SEGFAULT)
29 - metacall-test (Child aborted)
35 - metacall-distributable-test (Child aborted)
39 - metacall-inspect-test (Timeout)
40 - metacall-integration-test (SEGFAULT)
46 - metacall-python-open-test (SEGFAULT)

Currently class static method calls and object method calls are not properly handling errors.
If there's an uncaught exception in Ruby, the program will terminate (segfault).

See issue #53

To solve this, rb_protect must be used:
https://silverhammermba.github.io/emberb/c/#rb_protect

For implementing this loader, there is good support, so probably we can do it easily. Normally loaders can be fully implemented in C, but sometimes we can implement the loader in the target language itself with a small wrapper in C. This is useful where there isn't embedding API or if you want to do it in a simpler and faster way.

I have used the mock loader as a base for the dart loader, it provides a basic implementation as an example of what it should do.

As a documentation, we are going to use this:

• PoC: https://github.com/google/embed-dart-vm
• Reflection / Introspection: https://api.dart.dev/stable/2.9.0/dart-mirrors/dart-mirrors-library.html

Here is the list of tasks needed to implement it.

• Implement FindDart.cmake in order to find and/or install Dart and link it properly to the project (

  core/source/loaders/dart_loader/CMakeLists.txt

  Line 10 in a1ada51

  # TODO:

  ,

  core/source/loaders/dart_loader/CMakeLists.txt

  Line 145 in a1ada51

  # TODO:

  ,

  core/source/loaders/dart_loader/CMakeLists.txt

  Line 163 in a1ada51

  # TODO:

  ).
• Adapt the proof of concept to the dart loader and implement basic initialization and destruction mechanism.
• Implement a mechanism to load scripts from files.
• Implement a mechanism to introspect the loaded scripts. This can be implemented completely in Dart, but we should find a way to pass the handle loaded by the Dart VM by arguments to a Dart function and then operate with it (https://api.dart.dev/stable/2.9.0/dart-mirrors/dart-mirrors-library.html).
• Define the list of types (

  core/source/loaders/dart_loader/source/dart_loader_impl.cc

  Line 262 in a1ada51

  { TYPE_BOOL, "Boolean" },

  ).
• Implement type conversion functions between Dart types and MetaCall types (from MetaCall values to Dart values

  core/source/loaders/dart_loader/source/dart_loader_impl.cc

  Line 87 in a1ada51

  if (id == TYPE_BOOL)

  , and from Dart values to MetaCall values

  core/source/loaders/dart_loader/source/dart_loader_impl.cc

  Line 154 in a1ada51

  if (ret_type != NULL)

  ).
• Implement the call (it should be placed more or less here:

  core/source/loaders/dart_loader/source/dart_loader_impl.cc

  Line 153 in a1ada51
  , the original mock loader does not have any call because it is basically a mock).
• Implement an unit test with dart loader in order to verify it works properly.
• Implement an integration test with Flutter in order to demonstrate it works. We can do this in a new repository and for example use Flutter from NodeJS. Something similar to this: https://github.com/metacall/embedding-nodejs-example.

Travis fails after 50 minutes building because it only has one tasks (build all docker images) and it takes too long for the travis limits (https://docs.travis-ci.com/user/customizing-the-build/#build-timeouts).

This needs to be solved in order to allow proper builds (https://github.com/metacall/core/blob/develop/.travis.yml).

There are two possible solutions (maybe more):

1. Split all tasks in other jobs in order to avoid reaching the job timeout.
2. Use pulled images from docker or similar, and build with docker scripts.

Hello,

Current CLI shell doesn't support basic features like history, line editing, completion, etc. It can be easily achieved using the GNU Readline library.

I have attached to this issue an example of a tiny shell using it. Please, consider this as just an example, since it can contain memory leaks and vulnerabilities (e.g., user input is not properly checked).

Thank you.

readline_shell_template.tar.gz

This is the line that shows the segmentation fault:

Uncommenting it avoids the segmentation fault but probably generates leaks, because the handles (modules) are never cleared.

This is the test that generates the segmentation fault:
https://github.com/metacall/core/blob/627454ecae552918d2dbeee16414c64ceade800a/source/tests/metacall_ruby_object_class_test/source/metacall_ruby_object_class_test.cpp

When trying to compile with the latest version of cmake (3.20.0) it gives the following errors and warnings:

CMake Error at CMakeLists.txt:31 (string):
string no output variable specified

CMake Warning (dev) at CMakeLists.txt:81 (add_definitions):
Policy CMP0005 is not set: Preprocessor definition values are now escaped
automatically. Run "cmake --help-policy CMP0005" for policy details. Use
the cmake_policy command to set the policy and suppress this warning.
This warning is for project developers. Use -Wno-dev to suppress it.

CMake Warning (dev) at CMakeLists.txt:81 (add_definitions):
Policy CMP0005 is not set: Preprocessor definition values are now escaped
automatically. Run "cmake --help-policy CMP0005" for policy details. Use
the cmake_policy command to set the policy and suppress this warning.
This warning is for project developers. Use -Wno-dev to suppress it.

CMake Warning (dev) at CMakeLists.txt:81 (add_definitions):
Policy CMP0005 is not set: Preprocessor definition values are now escaped
automatically. Run "cmake --help-policy CMP0005" for policy details. Use
the cmake_policy command to set the policy and suppress this warning.
This warning is for project developers. Use -Wno-dev to suppress it.

CMake Warning (dev) at CMakeLists.txt:81 (add_definitions):
Policy CMP0005 is not set: Preprocessor definition values are now escaped
automatically. Run "cmake --help-policy CMP0005" for policy details. Use
the cmake_policy command to set the policy and suppress this warning.
This warning is for project developers. Use -Wno-dev to suppress it.

CMake Warning (dev) at CMakeLists.txt:81 (add_definitions):
Policy CMP0005 is not set: Preprocessor definition values are now escaped
automatically. Run "cmake --help-policy CMP0005" for policy details. Use
the cmake_policy command to set the policy and suppress this warning.
This warning is for project developers. Use -Wno-dev to suppress it.

-- Lib version
CMake Error at version/CMakeLists.txt:21 (include):
include could not find requested file:

Warnings

CMake Error at version/CMakeLists.txt:27 (include):
include could not find requested file:

SecurityFlags

CMake Error at version/CMakeLists.txt:33 (string):
string no output variable specified

CMake Error at version/CMakeLists.txt:34 (string):
string no output variable specified

CMake Error at version/CMakeLists.txt:57 (source_group_by_path):
Unknown CMake command "source_group_by_path".

CMake Warning (dev) in CMakeLists.txt:
No cmake_minimum_required command is present. A line of code such as

cmake_minimum_required(VERSION 3.20)

should be added at the top of the file. The version specified may be lower
if you wish to support older CMake versions for this project. For more
information run "cmake --help-policy CMP0000".
This warning is for project developers. Use -Wno-dev to suppress it.

I have created a placeholder for implementing PHP7 port with Swig: https://github.com/metacall/core/tree/develop/source/ports/php_port

We need to implement the following requirements:

• FindPHP7.cmake: A cmake script located in cmake folder in order to find PHP7 (I can do this).
• Interface implementing the functionality for calls, load, etc..: https://github.com/metacall/core/blob/develop/source/ports/php_port/interface/php_port/php_port_impl.i
• A working test with the proper include: https://github.com/metacall/core/blob/develop/source/ports/php_port/test/run.php.in
• Monkey Patching for require (optional). This should allow us to replace the require function and allow it to transparently import files from other languages (https://www.php.net/manual/en/function.override-function.php). This can be implemented in a php script wrapping the php extension:

I thought the example was not very helpful, because it doesn't even override the function with another function.
My question was: If I override a function, can I call the ORIGINAL function within the OVERRIDING function?
ie, can I do this:

<?php
override_function('strlen', '$string', 'return override_strlen($string);');
function override_strlen($string){
        return strlen($string);  
}
?>

The answer: NO, you will get a segfault.

HOWEVER, if you use rename_function to rename the original function to a third name, then call the third name in the OVERRIDING function, you will get the desired effect:

<?php
rename_function('strlen', 'new_strlen');
override_function('strlen', '$string', 'return override_strlen($string);');

function override_strlen($string){
        return new_strlen($string);  
}
?>

I plan to use this functionality to generate log reports every time a function is called, with the parameters, time, result, etc... So to wrap a function in logging, that was what I had to do.

• Deployment to composer (composer.json).

It showed some errors when I tried to build the visual studio project file made with cmake.

🐛 Bug Report

This message is shown in the metacall-node-port-test, it is generated by an exception of some instruction but it needs more debugging for finding what exact function generates that.

Expected Behavior

Not to show that message.

Current Behavior

The tests passes correctly but this message is printed instead. Maybe it's not important but it will be interesting to ensure what's happening.

Possible Solution

Not known yet. Needs investigation.

Steps to Reproduce

Run:

./docker-compose.sh build &> output

Then search for Start 40: metacall-node-port-test in the output file. Or:

make metacall-node-port-test
ctest -VV -R metacall-node-port-test

Context (Environment)

Here's the logs of the test. Note all exceptions show are correct, that's just the tests of the fail paths. The problem is "NodeJS Loader Error: Debug Failure.":

        Start  40: metacall-node-port-test

40: Test command: /usr/local/metacall/build/metacall-node-port-test
40: Environment variables: 
40:  LOADER_LIBRARY_PATH=/usr/local/metacall/build
40:  LOADER_SCRIPT_PATH=/usr/local/metacall/build/scripts
40:  CONFIGURATION_PATH=/usr/local/metacall/build/configurations/global.json
40:  SERIAL_LIBRARY_PATH=/usr/local/metacall/build
40:  DETOUR_LIBRARY_PATH=/usr/local/metacall/build
40:  PORT_LIBRARY_PATH=/usr/local/metacall/build
40: Test timeout computed to be: 1500
40: [==========] Running 1 test from 1 test case.
40: [----------] Global test environment set-up.
40: [----------] 1 test from metacall_node_port_test
40: [ RUN      ] metacall_node_port_test.DefaultConstructor
40: NodeJS Integration Test Passed
40: 
40: 
40:   metacall
40:     defined
40:       ✓ functions metacall and metacall_load_from_file must be defined
40:     fail
40: CSLoader static initialization
40: CSLoader static initialization
40: Could not find file '/usr/local/metacall/build/scripts/asd.invalid'.
40: NodeJS Loader Error: Debug Failure.
40: NodeJS Loader Error: Debug Failure.
40: NodeJS Loader Error: Debug Failure.
40: NodeJS Loader Error: Debug Failure.
40: { Error: Cannot find module './asd.invalid'
40:     at Function.Module._resolveFilename (internal/modules/cjs/loader.js:636:15)
40:     at Function.resolve (internal/modules/cjs/helpers.js:33:19)
40:     at Module.mod.require (/usr/local/metacall/build/node_modules/metacall/index.js:241:28)
40:     at Module.mod.require (/usr/local/metacall/source/ports/node_port/index.js:232:24)
40:     at require (internal/modules/cjs/helpers.js:25:18)
40:     at assert.throws (/usr/local/metacall/source/ports/node_port/test/index.js:47:26)
40:     at getActual (assert.js:567:5)
40:     at Function.throws (assert.js:679:24)
40:     at Context.it (/usr/local/metacall/source/ports/node_port/test/index.js:47:11)
40:     at callFn (/usr/local/metacall/source/ports/node_port/node_modules/mocha/lib/runnable.js:358:21) code: 'MODULE_NOT_FOUND' }
40: Could not find file '/usr/local/metacall/build/scripts/asd.invalid'.
40: Exception in ts_loader_trampoline_load_from_file { Error: ENOENT: no such file or directory, open '/usr/local/metacall/build/scripts/asd.invalid'
40:     at Object.openSync (fs.js:443:3)
40:     at Object.readFileSync (fs.js:343:35)
40:     at TypeScriptLanguageServiceHost.addFile (/usr/local/metacall/build/bootstrap.ts:136:19)
40:     at ts_loader_trampoline_load_from_file (/usr/local/metacall/build/bootstrap.ts:260:17)
40:     at metacall_require (/usr/local/metacall/source/ports/node_port/index.js:144:21)
40:     at Module.mod.require (/usr/local/metacall/source/ports/node_port/index.js:260:12)
40:     at require (internal/modules/cjs/helpers.js:25:18)
40:     at assert.throws (/usr/local/metacall/source/ports/node_port/test/index.js:47:26)
40:     at getActual (assert.js:567:5)
40:     at Function.throws (assert.js:679:24)
40:   errno: -2,
40:   syscall: 'open',
40:   code: 'ENOENT',
40:   path: '/usr/local/metacall/build/scripts/asd.invalid' }
40: { Error: Cannot find module './asd.invalid'
40:     at Function.Module._resolveFilename (internal/modules/cjs/loader.js:636:15)
40:     at Function.resolve (internal/modules/cjs/helpers.js:33:19)
40:     at Module.mod.require (/usr/local/metacall/source/ports/node_port/index.js:241:28)
40:     at require (internal/modules/cjs/helpers.js:25:18)
40:     at assert.throws (/usr/local/metacall/source/ports/node_port/test/index.js:47:26)
40:     at getActual (assert.js:567:5)
40:     at Function.throws (assert.js:679:24)
40:     at Context.it (/usr/local/metacall/source/ports/node_port/test/index.js:47:11)
40:     at callFn (/usr/local/metacall/source/ports/node_port/node_modules/mocha/lib/runnable.js:358:21)
40:     at Test.Runnable.run (/usr/local/metacall/source/ports/node_port/node_modules/mocha/lib/runnable.js:346:5) code: 'MODULE_NOT_FOUND' }
40: Could not find file '/usr/local/metacall/build/scripts/asd.cs'.
40: Exception in ts_loader_trampoline_load_from_file { Error: ENOENT: no such file or directory, open '/usr/local/metacall/build/scripts/asd.ts'
40:     at Object.openSync (fs.js:443:3)
40:     at Object.readFileSync (fs.js:343:35)
40:     at TypeScriptLanguageServiceHost.addFile (/usr/local/metacall/build/bootstrap.ts:136:19)
40:     at ts_loader_trampoline_load_from_file (/usr/local/metacall/build/bootstrap.ts:260:17)
40:     at metacall_require (/usr/local/metacall/source/ports/node_port/index.js:144:21)
40:     at Module.mod.require (/usr/local/metacall/source/ports/node_port/index.js:227:13)
40:     at require (internal/modules/cjs/helpers.js:25:18)
40:     at assert.throws (/usr/local/metacall/source/ports/node_port/test/index.js:52:26)
40:     at getActual (assert.js:567:5)
40:     at Function.throws (assert.js:679:24)
40:   errno: -2,
40:   syscall: 'open',
40:   code: 'ENOENT',
40:   path: '/usr/local/metacall/build/scripts/asd.ts' }
40: Exception in ts_loader_trampoline_load_from_file { Error: ENOENT: no such file or directory, open '/usr/local/metacall/build/scripts/asd.tsx'
40:     at Object.openSync (fs.js:443:3)
40:     at Object.readFileSync (fs.js:343:35)
40:     at TypeScriptLanguageServiceHost.addFile (/usr/local/metacall/build/bootstrap.ts:136:19)
40:     at ts_loader_trampoline_load_from_file (/usr/local/metacall/build/bootstrap.ts:260:17)
40:     at metacall_require (/usr/local/metacall/source/ports/node_port/index.js:144:21)
40:     at Module.mod.require (/usr/local/metacall/source/ports/node_port/index.js:227:13)
40:     at require (internal/modules/cjs/helpers.js:25:18)
40:     at assert.throws (/usr/local/metacall/source/ports/node_port/test/index.js:53:26)
40:     at getActual (assert.js:567:5)
40:     at Function.throws (assert.js:679:24)
40:   errno: -2,
40:   syscall: 'open',
40:   code: 'ENOENT',
40:   path: '/usr/local/metacall/build/scripts/asd.tsx' }
40:       ✓ require (507ms)
40:     load
40:       ✓ metacall_load_from_file (py)
40:       ✓ metacall_load_from_file (rb)
40:       ✓ metacall_load_from_memory (py)
40:       ✓ require (mock)
40:       ✓ require (py)
40:       ✓ require (py module)
40:       ✓ require (py submodule)
40:       ✓ require (rb)
40:     inspect
40: {"py":[{"name":"json.encoder","scope":{"name":"global_namespace","funcs":[{"name":"py_encode_basestring","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"s","type":{"name":"","id":18}}]},"async":false},{"name":"py_encode_basestring_ascii","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"s","type":{"name":"","id":18}}]},"async":false},{"name":"_make_iterencode","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"markers","type":{"name":"","id":18}},{"name":"_default","type":{"name":"","id":18}},{"name":"_encoder","type":{"name":"","id":18}},{"name":"_indent","type":{"name":"","id":18}},{"name":"_floatstr","type":{"name":"","id":18}},{"name":"_key_separator","type":{"name":"","id":18}},{"name":"_item_separator","type":{"name":"","id":18}},{"name":"_sort_keys","type":{"name":"","id":18}},{"name":"_skipkeys","type":{"name":"","id":18}},{"name":"_one_shot","type":{"name":"","id":18}},{"name":"ValueError","type":{"name":"","id":18}},{"name":"dict","type":{"name":"","id":18}},{"name":"float","type":{"name":"","id":18}},{"name":"id","type":{"name":"","id":18}},{"name":"int","type":{"name":"","id":18}},{"name":"isinstance","type":{"name":"","id":18}},{"name":"list","type":{"name":"","id":18}},{"name":"str","type":{"name":"","id":18}},{"name":"tuple","type":{"name":"","id":18}},{"name":"_intstr","type":{"name":"","id":18}}]},"async":false}],"classes":[{"name":"c_make_encoder"},{"name":"JSONEncoder"}],"objects":[]}},{"name":"helloworld.py","scope":{"name":"global_namespace","funcs":[{"name":"s_hello","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"s_sum","signature":{"ret":{"type":{"name":"int","id":4}},"args":[{"name":"left","type":{"name":"int","id":4}},{"name":"right","type":{"name":"int","id":4}}]},"async":false},{"name":"s_strcat","signature":{"ret":{"type":{"name":"str","id":7}},"args":[{"name":"left","type":{"name":"str","id":7}},{"name":"right","type":{"name":"str","id":7}}]},"async":false},{"name":"s_dont_load_this_function","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"left","type":{"name":"","id":18}},{"name":"right","type":{"name":"","id":18}}]},"async":false},{"name":"s_multiply","signature":{"ret":{"type":{"name":"int","id":4}},"args":[{"name":"left","type":{"name":"int","id":4}},{"name":"right","type":{"name":"int","id":4}}]},"async":false},{"name":"s_divide","signature":{"ret":{"type":{"name":"float","id":6}},"args":[{"name":"left","type":{"name":"float","id":6}},{"name":"right","type":{"name":"float","id":6}}]},"async":false}],"classes":[],"objects":[]}},{"name":"example.py","scope":{"name":"global_namespace","funcs":[{"name":"divide","signature":{"ret":{"type":{"name":"float","id":6}},"args":[{"name":"left","type":{"name":"float","id":6}},{"name":"right","type":{"name":"float","id":6}}]},"async":false},{"name":"hello","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"return_same_array","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"arr","type":{"name":"","id":18}}]},"async":false},{"name":"bytebuff","signature":{"ret":{"type":{"name":"bytes","id":8}},"args":[{"name":"input","type":{"name":"bytes","id":8}}]},"async":false},{"name":"dont_load_this_function","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"left","type":{"name":"","id":18}},{"name":"right","type":{"name":"","id":18}}]},"async":false},{"name":"sum","signature":{"ret":{"type":{"name":"int","id":4}},"args":[{"name":"left","type":{"name":"int","id":4}},{"name":"right","type":{"name":"int","id":4}}]},"async":false},{"name":"strcat","signature":{"ret":{"type":{"name":"str","id":7}},"args":[{"name":"left","type":{"name":"str","id":7}},{"name":"right","type":{"name":"str","id":7}}]},"async":false},{"name":"return_array","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"multiply","signature":{"ret":{"type":{"name":"int","id":4}},"args":[{"name":"left","type":{"name":"int","id":4}},{"name":"right","type":{"name":"int","id":4}}]},"async":false}],"classes":[],"objects":[]}},{"name":"html","scope":{"name":"global_namespace","funcs":[{"name":"unescape","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"s","type":{"name":"","id":18}}]},"async":false},{"name":"_replace_charref","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"s","type":{"name":"","id":18}}]},"async":false},{"name":"escape","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"s","type":{"name":"","id":18}},{"name":"quote","type":{"name":"","id":18}}]},"async":false}],"classes":[],"objects":[]}},{"name":"0x7f9a680011e0-0x7f9a85a22880-29-930707656","scope":{"name":"global_namespace","funcs":[{"name":"py_memory","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false}],"classes":[],"objects":[]}}],"rb":[{"name":"ducktype.rb","scope":{"name":"global_namespace","funcs":[{"name":"get_second","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"first","type":{"name":"","id":18}},{"name":"second","type":{"name":"","id":18}}]},"async":false},{"name":"map_style","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"a","type":{"name":"Fixnum","id":3}},{"name":"b","type":{"name":"Fixnum","id":3}}]},"async":false},{"name":"mixed","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"a","type":{"name":"","id":18}},{"name":"b","type":{"name":"","id":18}},{"name":"c","type":{"name":"Fixnum","id":3}},{"name":"d","type":{"name":"Fixnum","id":3}}]},"async":false},{"name":"say_null","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"say_multiply","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"left","type":{"name":"","id":18}},{"name":"right","type":{"name":"","id":18}}]},"async":false},{"name":"say_hello","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"value","type":{"name":"","id":18}}]},"async":false}],"classes":[],"objects":[]}},{"name":"cache.rb","scope":{"name":"global_namespace","funcs":[{"name":"cache_has_key","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"key","type":{"name":"String","id":7}}]},"async":false},{"name":"cache_set","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"key","type":{"name":"String","id":7}},{"name":"value","type":{"name":"String","id":7}}]},"async":false},{"name":"cache_get","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"key","type":{"name":"String","id":7}}]},"async":false},{"name":"cache_initialize","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"cache_nothing","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"key","type":{"name":"String","id":7}}]},"async":false}],"classes":[],"objects":[]}}],"cs":[],"ts":[],"node":[{"name":"bootstrap.ts","scope":{"name":"global_namespace","funcs":[{"name":"discover","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"handle","type":{"name":"","id":18}}]},"async":false},{"name":"await_function","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"func","type":{"name":"","id":18}},{"name":"args","type":{"name":"","id":18}},{"name":"trampoline_ptr","type":{"name":"","id":18}}]},"async":false},{"name":"load_from_file","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"paths","type":{"name":"","id":18}}]},"async":false},{"name":"initialize","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"await_future","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"future","type":{"name":"","id":18}},{"name":"trampoline_ptr","type":{"name":"","id":18}}]},"async":false},{"name":"test","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"obj","type":{"name":"","id":18}}]},"async":false},{"name":"load_from_package","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"clear","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"handle","type":{"name":"","id":18}}]},"async":false},{"name":"load_from_memory","signature":{"ret":{"type":{"name":"","id":18}},"args":[{"name":"name","type":{"name":"","id":18}},{"name":"buffer","type":{"name":"","id":18}},{"name":"opts","type":{"name":"","id":18}}]},"async":false},{"name":"destroy","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false},{"name":"execution_path","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":false}],"classes":[],"objects":[]}},{"name":"test.js","scope":{"name":"global_namespace","funcs":[{"name":"main","signature":{"ret":{"type":{"name":"","id":18}},"args":[]},"async":true}],"classes":[],"objects":[]}}],"mock":[{"name":"asd.mock","scope":{"name":"global_namespace","funcs":[{"name":"three_str","signature":{"ret":{"type":{"name":"String","id":7}},"args":[{"name":"a_str","type":{"name":"String","id":7}},{"name":"b_str","type":{"name":"String","id":7}},{"name":"c_str","type":{"name":"String","id":7}}]},"async":false},{"name":"my_empty_func_str","signature":{"ret":{"type":{"name":"String","id":7}},"args":[]},"async":false},{"name":"my_empty_func_int","signature":{"ret":{"type":{"name":"Integer","id":3}},"args":[]},"async":false},{"name":"new_args","signature":{"ret":{"type":{"name":"String","id":7}},"args":[{"name":"a_str","type":{"name":"String","id":7}}]},"async":false},{"name":"two_str","signature":{"ret":{"type":{"name":"String","id":7}},"args":[{"name":"a_str","type":{"name":"String","id":7}},{"name":"b_str","type":{"name":"String","id":7}}]},"async":false},{"name":"two_doubles","signature":{"ret":{"type":{"name":"Double","id":6}},"args":[{"name":"first_parameter","type":{"name":"Double","id":6}},{"name":"second_parameter","type":{"name":"Double","id":6}}]},"async":false},{"name":"my_empty_func","signature":{"ret":{"type":{"name":"Integer","id":3}},"args":[]},"async":false},{"name":"mixed_args","signature":{"ret":{"type":{"name":"Char","id":1}},"args":[{"name":"a_char","type":{"name":"Char","id":1}},{"name":"b_int","type":{"name":"Integer","id":3}},{"name":"c_long","type":{"name":"Long","id":4}},{"name":"d_double","type":{"name":"Double","id":6}},{"name":"e_ptr","type":{"name":"Ptr","id":11}}]},"async":false}],"classes":[],"objects":[]}}]}
40:       ✓ metacall_inspect
40:     call
40:       ✓ metacall (mock)
40:       ✓ metacall (py)
40:       ✓ metacall (rb)
40:     callback
40: 2 + 3 + 5 = 10
40: ------------------ js chain 5
40: ------------------ js chain pre x() call function(x)  ...
40: ------------------ js factorial 5
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 4
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 3
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 2
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 1
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 0
40: ------------------ js factorial case base
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js chain post x() call function(x)  ...
40: ------------------ js chain 5
40: ------------------ js chain pre x() call function(x)  ...
40: ------------------ js factorial 5
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 4
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 3
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 2
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 1
40: ------------------ js factorial pre x() call function(x)  ...
40: ------------------ js factorial 0
40: ------------------ js factorial case base
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js factorial post x() call function(x)  ...
40: ------------------ js chain post x() call function(x)  ...
40: 2  *  2  =  4
40: 4.0  /  2.0  =  2.0
40: 2  +  2  =  4
40: 2  +  2  =  22
40: 2  *  2  =  4
40: Executing: <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800e9720> ( 2.0 , 3.0 )
40: Returning lambda with captured arg: 10.0
40: Executing lambda with argument: 3.0
40: ------------------ py chain 5.0
40: ------------------ py chain pre x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dba0>
40: ------------------ py factorial 5.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800e9750>
40: ------------------ py factorial 4.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dcc0>
40: ------------------ py factorial 3.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dd80>
40: ------------------ py factorial 2.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003de40>
40: ------------------ py factorial 1.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003df00>
40: ------------------ py factorial 0.0
40: ------------------ py factorial case base
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003df00>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003de40>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dd80>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dcc0>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800e9750>
40: ------------------ py chain post x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dba0>
40: ------------------ py chain 5.0
40: ------------------ py chain pre x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dba0>
40: ------------------ py factorial 5.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800450c0>
40: ------------------ py factorial 4.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045180>
40: ------------------ py factorial 3.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045240>
40: ------------------ py factorial 2.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045300>
40: ------------------ py factorial 1.0
40: ------------------ py factorial pre x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800453c0>
40: ------------------ py factorial 0.0
40: ------------------ py factorial case base
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800453c0>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045300>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045240>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045180>
40: ------------------ py factorial post x() call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a800450c0>
40: ------------------ py chain post x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a8003dba0>
40: ------------------ py chain 5.0
40: ------------------ py chain pre x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045540>
40: ------------------ js factorial 5
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 4
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 3
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 2
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 1
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 0
40: ------------------ js factorial case base
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ py chain post x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045540>
40: ------------------ py chain 5.0
40: ------------------ py chain pre x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045540>
40: ------------------ js factorial 5
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 4
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 3
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 2
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 1
40: ------------------ js factorial pre x() call function ()  ...
40: ------------------ js factorial 0
40: ------------------ js factorial case base
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ js factorial post x() call function ()  ...
40: ------------------ py chain post x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045540>
40: ------------------ py chain 5.0
40: ------------------ py chain pre x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045630>
40: ------------------ py chain post x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045630>
40: ------------------ py chain 5.0
40: ------------------ py chain pre x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045630>
40: ------------------ py chain post x call <built-in method __py_loader_impl_function_type_invoke__ of PyCapsule object at 0x7f9a80045630>
40:       ✓ callback (py) (133ms)
40: 
40: 
40:   15 passing (684ms)
40: 
40: ruby: set value
40: ruby: get value
40: Second value is
40: 12.0
40: [       OK ] metacall_node_port_test.DefaultConstructor (1343 ms)
40: [----------] 1 test from metacall_node_port_test (1343 ms total)
40: 
40: [----------] Global test environment tear-down
40: [==========] 1 test from 1 test case ran. (1344 ms total)
40: [  PASSED  ] 1 test.
 40/111 Test  #40: metacall-node-port-test ..........................   Passed    1.35 sec

Detailed Description

None.

Possible Implementation

None.

The idea is to fire up a Node REPL when the CLI is initiated.
The REPL will parse commands and arguments and match them with a list of available commands (plugins), that will be stored in individual files (possibly with the same name as the function they contain).

This is a Proof of Concept for Crystal Loader. The idea is to allow it to be fully implemented in Crystal lang, with a minimal C wrapper: crystal_lib.zip. It is based on this repository: https://github.com/ysbaddaden/crystal_library (with minimal modifications and the addition of the Dockerfile).

In order to build and run the test, use Docker or read the README.md. For Docker:

docker build -t metacall/crystal .

The TODO list is the following:

• Implement a CMake file in the cmake folder for finding Crystal folders (and installing CMake script for Crystal can be a plus, but not needed). The following paths are needed:

CRYSTAL = /usr/lib/crystal/bin/crystal
CRYSTAL_SRC = /usr/share/crystal/src
CRYSTAL_LIB = /usr/lib/crystal/lib

Also finding libcrystal.a and libgc.a which are located at $(CRYSTAL_SRC)/ext/libcrystal.a and $(CRYSTAL_LIB)/libgc.a respectively.

Then we must implement the build and linkage of the library as in the Makefile but for CMake, it can be done inside the same CMake script with a function or ad-hoc in the CMakeLists.txt of the cr_loader.

• Implement the build with the previous CMake script in the cr_loader CMakeLists.txt. We must build the cr_loader as a dynamic library, linked with all dependencies (static libs from Crystal and dynamic libs like pcre).

• Implement a test called cr-loader-test (in folder source/tests/cr_loader_test) for testing the cr_loader standalone.

• Implement a test called metacall-crystal-test (in folder source/tests/metacall_crystal_test) with a Crystal script in order to do a integration test with Crystal and MetaCall.

• Implement the bindings for MetaCall library in Crystal lang for the functions that will be used in Crystal. The minimal set of headers needed for a plugin are:

#include <loader/loader_impl.h>
#include <loader/loader_path.h>

#include <reflect/reflect_type.h>
#include <reflect/reflect_function.h>
#include <reflect/reflect_scope.h>
#include <reflect/reflect_context.h>

#include <log/log.h>

For the C bindings we can:

1. Do them manually: https://crystal-lang.org/reference/syntax_and_semantics/c_bindings/
2. Do them automatically: https://github.com/crystal-lang/crystal_lib (but this seems experimental and less explicit).

• Implement cr_loader functionality in Crystal, using the bindings from the previous step (TODO: Extend this point later on with more design and implementation details like AST or Compiler Internals).

• metacall/core has Travis-ci integrated. There could be a badge added to README.md
• .github/ISSUE_TEMPLATE and .github/PULL_REQUEST_TEMPLATE are blank and need to be set

@giarve please check if these changes can be added to develop

I went through building steps here and I got this error and can't find solution.

build git:(develop) cmake --build . --target install

Scanning dependencies of target version
[  1%] Building C object source/version/CMakeFiles/version.dir/__/metacall/source/metacall_version.c.o
[  1%] Built target version
Scanning dependencies of target preprocessor
[  1%] Building C object source/preprocessor/CMakeFiles/preprocessor.dir/source/preprocessor.c.o
[  1%] Built target preprocessor
Scanning dependencies of target environment
[  1%] Building C object source/environment/CMakeFiles/environment.dir/source/environment.c.o
[  2%] Building C object source/environment/CMakeFiles/environment.dir/source/environment_variable.c.o
[  2%] Building C object source/environment/CMakeFiles/environment.dir/source/environment_variable_path.c.o
[  2%] Built target environment
Scanning dependencies of target format
[  2%] Building C object source/format/CMakeFiles/format.dir/source/format.c.o
[  3%] Building C object source/format/CMakeFiles/format.dir/source/format_print.c.o
[  3%] Built target format
Scanning dependencies of target threading
[  4%] Building C object source/threading/CMakeFiles/threading.dir/source/threading.c.o
[  4%] Building C object source/threading/CMakeFiles/threading.dir/source/threading_thread_id.c.o
/Users/omarkhaled/core/source/threading/source/threading_thread_id.c:77:4: error: implicit declaration of function
      'pthread_threadid_np' is invalid in C99 [-Werror,-Wimplicit-function-declaration]
                        pthread_threadid_np(NULL, &thread_id);
                        ^
1 error generated.
gmake[2]: *** [source/threading/CMakeFiles/threading.dir/build.make:95: source/threading/CMakeFiles/threading.dir/source/threading_thread_id.c.o] Error 1
gmake[1]: *** [CMakeFiles/Makefile2:3601: source/threading/CMakeFiles/threading.dir/all] Error 2
gmake: *** [Makefile:171: all] Error 2