Skip to content

Commit

Permalink
fix mcxcloud job max duration bug, kill runtime>1min
Browse files Browse the repository at this point in the history
  • Loading branch information
@fangq
fangq committed Apr 15, 2023
1 parent 70b3b5e commit 7515611
Show file tree
Hide file tree
Showing 5 changed files with 28 additions and 20 deletions.
5 changes: 3 additions & 2 deletions mcxcloud/backend/mcxcloudd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,18 @@ $savetime=time();
# delete all jobs running over 1 min, if --kill is attached

if(grep(/--kill/,@ARGV)){
my @joblist=`docker service ls | grep 'fangqq/mcx:v20'| cut -f 1 -d" " | xargs docker service ps | grep 'Running [0-9]* minute' | cut -f 1,4 -d" "`;
my @joblist=`docker service ls | grep 'fangqq/mcx:v20'| cut -f 1 -d" " | xargs docker service ps | grep 'Running [0-9a-z ]* minute' | cut -f 1,4 -d" "`;
chomp @joblist;
if(@joblist>0){
my @jobid;
foreach my $job (@joblist){
if($job=~/^(\w+)\s+(\w+)\./){
push(@jobid,$1);
push(@jobid,$2);
system("touch workspace/$2/killed");
print "killed overtime job: $2\n";
}
}
print "docker service rm ".join(' ',@jobid)."\n";
system("docker service rm ".join(' ',@jobid)) if(@jobid>0);
}
exit;
Expand Down
35 changes: 21 additions & 14 deletions mcxcloud/backend/mcxserver.cgi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ die if(not ($ENV{"HTTP_REFERER"} =~ /^https*:\/\/mcx\.space/));

my ($DBName,$DBUser,$DBPass,%DBErr,$dbh,$sth,$html,$page,$jobid,$savetime,$dbname,$md5key,$callback,$jobhash);
my $q = new CGI;
my %jobstatus=(0=>'queued',1=>'initiated',2=>'created',3=>'running',4=>'completed',5=>'deleted',6=>'failed',7=>'invalid',8=>'cancelled', 9=>'writing output', 10=>'use cached data');
my %jobstatus=(0=>'queued',1=>'initiated',2=>'created',3=>'running',4=>'completed',5=>'deleted',6=>'failed',7=>'invalid',8=>'cancelled', 9=>'writing output', 10=>'use cached data', 11=>'killed');

# Default paths for storage of the job/simulation database and simulation work folders
# both folders must be symbolic links pointing to network-shared folders via NSF that
Expand Down Expand Up @@ -208,11 +208,17 @@ if(&V("hash") ne '' && &V("id") ne ''){ # loading simulation JSON when one clic
chomp(my @lines = <FF>);
close(FF);
$html =$callback.'({"status":"'.$jobstatus{$status}.'","jobid":"'.$jobid.'", "dberror": "'.join(/\n/,@lines).'"})'."\n";
}elsif(-e "$workspace/$jobid/input.json" && not -z "$workspace/$jobid/input.json"){ # when a job is started, the input file is written to the work folder
$status=2;
$html =$callback.'({"status":"'.$jobstatus{$status}.'","jobid":"'.$jobid.'"})'."\n";
}elsif(-e "$workspace/$jobid/killed"){ # job is killed
$status=11;
$html =$callback.'({"status":"'.$jobstatus{$status}.'","jobid":"'.$jobid.'", "dberror": "exceed max runtime (1 min)"})'."\n";
}elsif($jobid ne '' && -d "$workspace/$jobid"){ # when a job folder is just created, but no input file is written
$status=1;
if(-e "$workspace/$jobid/input.json" && not -z "$workspace/$jobid/input.json"){ # when a job is started, the input file is written $
$status=2;
}
if(not -e "$workspace/$jobid/done" && -e "$workspace/$jobid/output.log" && not -z "$workspace/$jobid/output.log"){ # job is running
$status=3;
}
$html =$callback.'({"status":"'.$jobstatus{$status}.'","jobid":"'.$jobid.'"})'."\n";
}else{ # use the database to inqure the last updated status
my $cmd="select status from $dbname where jobid='$jobid';";
Expand All @@ -239,19 +245,19 @@ sub V{
sub checklimit{
my ($req)=@_;
my $html;
if($req->{'Session'}->{'Photons'}>1e7){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the max photon number is limited to 1e7 in this preview version"})'."\n";
if($req->{'Session'}->{'Photons'}>5e8){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the max photon number is limited to 5e8 in this preview version"})'."\n";
}
if($html eq '' && defined($req->{'Session'}->{'DebugFlag'}) && $req->{'Session'}->{'DebugFlag'} =~ /m/i){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"storing photon trajectories is not supported in this preview version"})'."\n";
}
if($html eq '' && ($req->{'Forward'}->{'T1'}/$req->{'Forward'}->{'Dt'})>20){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the maximum time gate number is limited to 20 in this preview version"})'."\n";
if($html eq '' && ($req->{'Forward'}->{'T1'}/$req->{'Forward'}->{'Dt'})>100){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the maximum time gate number is limited to 100 in this preview version"})'."\n";
}
if($html eq '' && @{$req->{'Domain'}->{'Dim'}} == 3){
foreach my $len (@{$req->{'Domain'}->{'Dim'}}){
if($len>100){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the maximum domain dimension is 100 in this preview version"})'."\n";
if($len>300){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the maximum domain dimension is 300 in this preview version"})'."\n";
last;
}
}
Expand All @@ -260,8 +266,8 @@ sub checklimit{
foreach my $obj (@{$req->{'Shapes'}}){
if(defined($obj->{'Grid'})){
foreach my $len (@{$obj->{'Grid'}->{'Size'}}){
if($len>100){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the maximum domain dimension is 100 in this preview version"})'."\n";
if($len>300){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"the maximum domain dimension is 300 in this preview version"})'."\n";
last;
}
}
Expand All @@ -271,8 +277,8 @@ sub checklimit{
}
if($html eq '' && @{$req->{'Domain'}->{'Media'}} > 1){
foreach my $obj (@{$req->{'Domain'}->{'Media'}}){
if(defined($obj->{'mus'}) && $obj->{'mus'}>20){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"scattering coeff (mus) is limited to 20/mm in this preview version"})'."\n";
if(defined($obj->{'mus'}) && $obj->{'mus'}>50){
$html =$callback.'({"status":"invalid","jobid":"'.$jobid.'","dberror":"scattering coeff (mus) is limited to 50/mm in this preview version"})'."\n";
last;
}
}
Expand All @@ -283,3 +289,4 @@ sub checklimit{
exit;
}
}

4 changes: 2 additions & 2 deletions mcxcloud/frontend/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2255,7 +2255,7 @@ <h3>Backend</h3>
function updatestatus(data){
let d = new Date();
if(data.hasOwnProperty('status') && data.hasOwnProperty('dberror')){
if(data.status === 'invalid'){
if(data.status === 'invalid' || data.status === 'killed'){
$("#run-log").val($("#run-log").val()+"\n"+d.toLocaleTimeString()+" "+data.status+":"+data.dberror);
resetjob();
return;
Expand Down Expand Up @@ -2291,7 +2291,7 @@ <h3>Backend</h3>
$("#run-log").val($("#run-log").val()+"\n"+d.toLocaleTimeString()+" "+data.status+":"+data.dberror);
else
$("#run-log").val($("#run-log").val()+"\n"+d.toLocaleTimeString()+" "+data.status);
if(data.status === 'invalid' || data.status === 'failed')
if(data.status === 'invalid' || data.status === 'failed' || data.status === 'killed')
resetjob();
}
if(data.hasOwnProperty('log')){
Expand Down
2 changes: 1 addition & 1 deletion pymcx
Submodule pymcx updated 3 files
+1 −2 pymcx/__init__.py
+6 −13 pymcx/mcx_run.py
+0 −26 pymcx/mcx_writevolbin.py
2 changes: 1 addition & 1 deletion src/pybind11
Submodule pybind11 updated 79 files
+65 −61 .clang-tidy
+5 −3 .github/CONTRIBUTING.md
+0 −4 .github/dependabot.yml
+40 −23 .github/workflows/ci.yml
+1 −1 .github/workflows/configure.yml
+6 −4 .github/workflows/format.yml
+7 −5 .github/workflows/pip.yml
+3 −3 .github/workflows/upstream.yml
+14 −11 .pre-commit-config.yaml
+13 −0 CMakeLists.txt
+3 −0 docs/_static/css/custom.css
+0 −11 docs/_static/theme_overrides.css
+2 −2 docs/advanced/cast/custom.rst
+3 −3 docs/advanced/cast/stl.rst
+2 −2 docs/advanced/classes.rst
+2 −2 docs/advanced/pycpp/numpy.rst
+1 −1 docs/advanced/smart_ptrs.rst
+160 −12 docs/changelog.rst
+1 −1 docs/classes.rst
+6 −17 docs/conf.py
+ − docs/pybind11-logo.png
+4 −3 docs/requirements.txt
+5 −3 include/pybind11/attr.h
+5 −5 include/pybind11/cast.h
+23 −30 include/pybind11/detail/class.h
+21 −2 include/pybind11/detail/common.h
+1 −1 include/pybind11/detail/init.h
+1 −0 include/pybind11/detail/internals.h
+0 −66 include/pybind11/detail/type_caster_base.h
+9 −3 include/pybind11/detail/typeid.h
+17 −6 include/pybind11/eigen.h
+56 −32 include/pybind11/embed.h
+2 −2 include/pybind11/functional.h
+1 −1 include/pybind11/iostream.h
+19 −8 include/pybind11/numpy.h
+44 −26 include/pybind11/pybind11.h
+338 −64 include/pybind11/pytypes.h
+6 −6 include/pybind11/stl.h
+16 −6 noxfile.py
+2 −1 pybind11/__init__.py
+8 −1 pybind11/__main__.py
+1 −1 pybind11/_version.py
+12 −0 pybind11/commands.py
+2 −1 setup.cfg
+1 −0 setup.py
+1 −0 tests/CMakeLists.txt
+14 −1 tests/conftest.py
+51 −0 tests/cross_module_interleaved_error_already_set.cpp
+68 −58 tests/extra_python_package/test_files.py
+25 −0 tests/pybind11_tests.cpp
+1 −1 tests/requirements.txt
+8 −2 tests/test_builtin_casters.cpp
+2 −2 tests/test_constants_and_functions.cpp
+7 −7 tests/test_custom_type_casters.cpp
+5 −2 tests/test_eigen.cpp
+7 −0 tests/test_eigen.py
+1 −1 tests/test_embed/CMakeLists.txt
+47 −2 tests/test_exceptions.cpp
+99 −3 tests/test_exceptions.py
+10 −1 tests/test_kwargs_and_defaults.cpp
+21 −9 tests/test_methods_and_attributes.py
+2 −0 tests/test_modules.cpp
+30 −0 tests/test_modules.py
+1 −1 tests/test_numpy_array.cpp
+1 −1 tests/test_numpy_dtypes.cpp
+1 −1 tests/test_numpy_vectorize.cpp
+183 −0 tests/test_pytypes.cpp
+129 −0 tests/test_pytypes.py
+2 −2 tests/test_smart_ptr.cpp
+15 −8 tests/test_stl.cpp
+2 −2 tests/test_tagbased_polymorphic.cpp
+2 −3 tests/test_virtual_functions.cpp
+21 −11 tools/FindPythonLibsNew.cmake
+23 −0 tools/JoinPaths.cmake
+7 −0 tools/pybind11.pc.in
+3 −1 tools/pybind11NewTools.cmake
+32 −11 tools/pybind11Tools.cmake
+2 −0 tools/setup_global.py.in
+2 −0 tools/setup_main.py.in

0 comments on commit 7515611

Please sign in to comment.