API Documentation

The following is an extract of the most important parts of the API of DALiuGE. For a complete reference please go to the source code.

• dlg
  • dlg.ddap_protocol
    • AppDROPStates
    • ChecksumTypes
    • DROPLinkType
    • DROPPhases
    • DROPRel
      • DROPRel.lhs
      • DROPRel.rel
      • DROPRel.rhs
    • DROPStates
    • ExecutionMode
  • dlg.drop
    • AbstractDROP
      • AbstractDROP.addConsumer()
      • AbstractDROP.addProducer()
      • AbstractDROP.addStreamingConsumer()
      • AbstractDROP.autofill_environment_variables()
      • AbstractDROP.cancel()
      • AbstractDROP.commit()
      • AbstractDROP.completedrop()
      • AbstractDROP.consumers
      • AbstractDROP.dlg_session_id
      • AbstractDROP.dropReproComplete()
      • AbstractDROP.executionMode
      • AbstractDROP.generate_merkle_data()
      • AbstractDROP.generate_recompute_data()
      • AbstractDROP.generate_repeat_data()
      • AbstractDROP.generate_replicate_comp_data()
      • AbstractDROP.generate_replicate_sci_data()
      • AbstractDROP.generate_replicate_total_data()
      • AbstractDROP.generate_reproduce_data()
      • AbstractDROP.generate_rerun_data()
      • AbstractDROP.get_consumers_nodes()
      • AbstractDROP.get_environment_variable()
      • AbstractDROP.get_environment_variables()
      • AbstractDROP.handleEvent()
      • AbstractDROP.handleInterest()
      • AbstractDROP.humanKey
      • AbstractDROP.initialize()
      • AbstractDROP.isCompleted()
      • AbstractDROP.log_level
      • AbstractDROP.oid
      • AbstractDROP.parent
      • AbstractDROP.persist
      • AbstractDROP.phase
      • AbstractDROP.producerFinished()
      • AbstractDROP.producers
      • AbstractDROP.setCompleted()
      • AbstractDROP.setError()
      • AbstractDROP.size
      • AbstractDROP.skip()
      • AbstractDROP.status
      • AbstractDROP.streamingConsumers
      • AbstractDROP.uid
    • DROPLogFilter
      • DROPLogFilter.filter()
    • InstanceLogHandler
      • InstanceLogHandler.emit()
    • ListAsDict
      • ListAsDict.append()
  • dlg.droputils
    • DROPFile
    • DROPWaiterCtx
    • EvtConsumer
    • allDropContents()
    • breadFirstTraverse()
    • copyDirectoryContents()
    • copyDropContents()
    • depthFirstTraverse()
    • getDownstreamObjects()
    • getLeafNodes()
    • getUpstreamObjects()
    • has_path()
    • listify()
    • replace_dataurl_placeholders()
    • replace_placeholders()
  • dlg.event
    • Event
    • EventFirer
      • EventFirer.subscribe()
      • EventFirer.unsubscribe()
    • EventHandler
  • dlg.graph_loader
    • addLink()
    • loadDropSpecs()
  • dlg.rpc
    • DropProxy
      • DropProxy.proxy_info
    • ProxyInfo
    • RPCClient
    • RPCClientBase
    • RPCObject
    • RPCServer
    • RPCServerBase
    • ZeroRPCClient
      • ZeroRPCClient.request
        • ZeroRPCClient.request.args
        • ZeroRPCClient.request.method
        • ZeroRPCClient.request.queue
      • ZeroRPCClient.response
        • ZeroRPCClient.response.is_exception
        • ZeroRPCClient.response.value
    • ZeroRPCServer
  • dlg.runtime.delayed
    • delayed()
  • dlg.utils
    • ExistingProcess
      • ExistingProcess.kill()
      • ExistingProcess.poll()
      • ExistingProcess.terminate()
      • ExistingProcess.wait()
    • ZlibUncompressedStream
    • browse_service()
    • createDirIfMissing()
    • deregister_service()
    • escapeQuotes()
    • fname_to_pipname()
    • getDlgDir()
    • getDlgLogsDir()
    • getDlgPath()
    • getDlgPidDir()
    • getDlgVariable()
    • getDlgWorkDir()
    • get_all_ipv4_addresses()
    • get_local_ip_addr()
    • get_symbol()
    • isabs()
    • object_tracking()
    • prepare_sql()
    • register_service()
    • timed_import()
    • to_externally_contactable_host()
    • truncateUidToKey()
    • zmq_safe()
• dlg.manager
  • dlg.manager.session
  • dlg.manager.drop_manager
    • DROPManager
      • DROPManager.addGraphSpec()
      • DROPManager.cancelSession()
      • DROPManager.createSession()
      • DROPManager.deploySession()
      • DROPManager.destroySession()
      • DROPManager.getGraph()
      • DROPManager.getGraphReproData()
      • DROPManager.getGraphSize()
      • DROPManager.getGraphStatus()
      • DROPManager.getSessionIds()
      • DROPManager.getSessionReproStatus()
      • DROPManager.getSessionStatus()
  • dlg.manager.node_manager
    • ErrorStatusListener
    • EventMixIn
    • NodeManager
    • NodeManagerBase
      • NodeManagerBase.addGraphSpec()
      • NodeManagerBase.cancelSession()
      • NodeManagerBase.createSession()
      • NodeManagerBase.deliver_event()
      • NodeManagerBase.deploySession()
      • NodeManagerBase.destroySession()
      • NodeManagerBase.getGraph()
      • NodeManagerBase.getGraphReproData()
      • NodeManagerBase.getGraphSize()
      • NodeManagerBase.getGraphStatus()
      • NodeManagerBase.getSessionIds()
      • NodeManagerBase.getSessionReproStatus()
      • NodeManagerBase.getSessionStatus()
      • NodeManagerBase.manage_session_failure()
    • NodeManagerDropRunner
    • NodeManagerProcessDropRunner
      • NodeManagerProcessDropRunner.run_drop()
    • NodeManagerThreadDropRunner
      • NodeManagerThreadDropRunner.run_drop()
    • RpcMixIn
    • ZMQPubSubMixIn
      • ZMQPubSubMixIn.subscription
        • ZMQPubSubMixIn.subscription.endpoint
        • ZMQPubSubMixIn.subscription.finished_evt
  • dlg.manager.composite_manager
    • CompositeManager
      • CompositeManager.addGraphSpec()
      • CompositeManager.cancelSession()
      • CompositeManager.createSession()
      • CompositeManager.deploySession()
      • CompositeManager.destroySession()
      • CompositeManager.getGraph()
      • CompositeManager.getGraphReproData()
      • CompositeManager.getGraphSize()
      • CompositeManager.getGraphStatus()
      • CompositeManager.getPastSessionIds()
      • CompositeManager.getSessionDir()
      • CompositeManager.getSessionIds()
      • CompositeManager.getSessionReproStatus()
      • CompositeManager.getSessionStatus()
      • CompositeManager.get_node_from_json()
      • CompositeManager.replicate()
    • DataIslandManager
      • DataIslandManager.getGraphReproData()
      • DataIslandManager.getSessionReproStatus()
    • MasterManager
      • MasterManager.getGraphReproData()
      • MasterManager.getSessionReproStatus()
    • sanitize_link()
  • dlg.manager.rest
    • CompositeManagerRestServer
      • CompositeManagerRestServer.addCMNode()
      • CompositeManagerRestServer.getCMNodes()
      • CompositeManagerRestServer.getCMStatus()
      • CompositeManagerRestServer.getNodeSessions()
      • CompositeManagerRestServer.getPastSessions()
      • CompositeManagerRestServer.initializeSpecifics()
      • CompositeManagerRestServer.pastSessions()
      • CompositeManagerRestServer.removeCMNode()
    • ManagerRestServer
      • ManagerRestServer.getDropData()
      • ManagerRestServer.initializeSpecifics()
    • MasterManagerRestServer
      • MasterManagerRestServer.initializeSpecifics()
    • NMRestServer
      • NMRestServer.initializeSpecifics()
  • dlg.manager.client
• dlg.apps
  • dlg.apps.app_base
    • AppDROP
      • AppDROP.cancel()
      • AppDROP.dataWritten()
      • AppDROP.dropCompleted()
      • AppDROP.drop_runner
      • AppDROP.execStatus
      • AppDROP.handleEvent()
      • AppDROP.initialize()
      • AppDROP.inputs
      • AppDROP.outputs
      • AppDROP.skip()
      • AppDROP.streamingInputs
    • BarrierAppDROP
      • BarrierAppDROP.initialize()
    • DropRunner
      • DropRunner.run_drop()
    • InputFiredAppDROP
      • InputFiredAppDROP.dropCompleted()
      • InputFiredAppDROP.execute()
      • InputFiredAppDROP.exists()
      • InputFiredAppDROP.initialize()
    • SyncDropRunner
      • SyncDropRunner.run_drop()
    • run_on_daemon_thread()
  • dlg.apps.bash_shell_app
    • BashShellApp
    • BashShellBase
    • StreamingInputBashApp
    • StreamingInputBashAppBase
      • StreamingInputBashAppBase.dataWritten()
      • StreamingInputBashAppBase.dropCompleted()
      • StreamingInputBashAppBase.initialize()
    • StreamingInputOutputBashApp
    • StreamingOutputBashApp
    • prepare_input_channel()
    • prepare_output_channel()
  • dlg.apps.branch
    • BranchAppDrop
      • BranchAppDrop.execute()
  • dlg.apps.constructs
    • CommentDrop
    • ExclusiveForceDrop
    • GatherDrop
    • GlobalVariableDrop
    • GroupByDrop
    • LoopDrop
    • MKNDrop
    • ScatterDrop
    • SubGraphDrop
  • dlg.apps.dynlib
    • CDlgApp
    • CDlgInput
    • CDlgOutput
    • CDlgStreamingInput
    • DynlibApp
      • DynlibApp.generate_recompute_data()
      • DynlibApp.initialize()
    • DynlibProcApp
      • DynlibProcApp.cancel()
      • DynlibProcApp.initialize()
    • DynlibStreamApp
      • DynlibStreamApp.dataWritten()
      • DynlibStreamApp.dropCompleted()
      • DynlibStreamApp.generate_recompute_data()
      • DynlibStreamApp.initialize()
    • FinishSubprocess
    • InvalidLibrary
    • get_from_subprocess()
    • load_and_init()
    • prepare_c_inputs()
    • prepare_c_outputs()
    • prepare_c_ranks()
    • run()
  • dlg.apps.mpi
    • MPIApp
      • MPIApp.generate_recompute_data()
      • MPIApp.initialize()
  • dlg.apps.pyfunc
    • PyFuncApp
      • PyFuncApp.generate_recompute_data()
      • PyFuncApp.initialize()
      • PyFuncApp.initialize_with_func_code()
      • PyFuncApp.run()
    • PyMemberApp
    • import_using_code()
    • import_using_code_ser()
  • dlg.apps.dockerapp
    • ContainerIpWaiter
    • DockerApp
      • DockerApp.generate_recompute_data()
      • DockerApp.handleInterest()
      • DockerApp.initialize()
    • DockerPath
      • DockerPath.path
  • dlg.apps.simple
    • ArrayGatherApp
      • ArrayGatherApp.initialize()
    • AverageArraysApp
      • AverageArraysApp.getInputArrays()
      • AverageArraysApp.initialize()
    • Branch
      • Branch.write_results()
    • CopyApp
    • DALiuGEApp
    • DictGatherApp
      • DictGatherApp.initialize()
    • GenericGatherApp
    • GenericNpyGatherApp
      • GenericNpyGatherApp.gather_inputs()
      • GenericNpyGatherApp.reduce_gather_inputs()
    • GenericNpyScatterApp
    • GenericScatterApp
    • HelloWorldApp
    • ListAppendThrashingApp
      • ListAppendThrashingApp.initialize()
    • NullBarrierApp
      • NullBarrierApp.component_meta
    • PickOne
      • PickOne.initialize()
      • PickOne.writeData()
    • RandomArrayApp
      • RandomArrayApp.initialize()
    • SimpleBranch
      • SimpleBranch.initialize()
    • SleepAndCopyApp
    • SleepApp
    • UrlRetrieveApp
  • dlg.apps.socket_listener
    • SocketListenerApp
  • dlg.apps.scp
    • ScpApp
      • ScpApp.initialize()
  • dlg.apps.archiving
    • ExternalStoreApp
      • ExternalStoreApp.store()
    • NgasArchivingApp
      • NgasArchivingApp.store()
  • dlg.apps.crc
    • CRCApp
    • CRCStreamApp
      • CRCStreamApp.dataWritten()
      • CRCStreamApp.dropCompleted()
      • CRCStreamApp.initialize()
• dlg.data
  • dlg.data.io
    • DataIO
      • DataIO.buffer()
      • DataIO.close()
      • DataIO.delete()
      • DataIO.exists()
      • DataIO.isOpened()
      • DataIO.open()
      • DataIO.read()
      • DataIO.size()
      • DataIO.write()
    • DirectoryIO
      • DirectoryIO.getDirName()
    • ErrorIO
    • FileIO
      • FileIO.getFileName()
    • IOForURL()
    • MemoryIO
    • NgasIO
      • NgasIO.delete()
    • NgasLiteIO
      • NgasLiteIO.exists()
    • NullIO
    • OpenMode
    • SharedMemoryIO
    • total_dir_size()
  • dlg.data.drops.data_base
    • DataDROP
      • DataDROP.checksum
      • DataDROP.checksumType
      • DataDROP.close()
      • DataDROP.dataURL
      • DataDROP.decrRefCount()
      • DataDROP.delete()
      • DataDROP.exists()
      • DataDROP.getIO()
      • DataDROP.getLogs()
      • DataDROP.incrRefCount()
      • DataDROP.isBeingRead()
      • DataDROP.open()
      • DataDROP.read()
      • DataDROP.write()
    • DataDROPError
    • EndDROP
    • NullDROP
      • NullDROP.dataURL
      • NullDROP.getIO()
    • PathBasedDrop
      • PathBasedDrop.get_dir()
  • dlg.data.drops.directory
    • DirectoryDROP
      • DirectoryDROP.dataURL
      • DirectoryDROP.delete()
      • DirectoryDROP.exists()
      • DirectoryDROP.getIO()
      • DirectoryDROP.initialize()
  • dlg.data.drops.file
    • FileDROP
      • FileDROP.dataURL
      • FileDROP.delete()
      • FileDROP.generate_reproduce_data()
      • FileDROP.getIO()
      • FileDROP.initialize()
      • FileDROP.sanitize_paths()
      • FileDROP.setCompleted()
  • dlg.data.drops.memory
    • InMemoryDROP
      • InMemoryDROP.buftype
      • InMemoryDROP.dataURL
      • InMemoryDROP.generate_reproduce_data()
      • InMemoryDROP.getIO()
      • InMemoryDROP.initialize()
    • PythonObjectDROP
    • SharedMemoryDROP
      • SharedMemoryDROP.buftype
      • SharedMemoryDROP.dataURL
      • SharedMemoryDROP.getIO()
      • SharedMemoryDROP.initialize()
    • get_builtins()
    • guess_type()
    • parse_pydata()
  • dlg.data.drops.ngas
    • NgasDROP
      • NgasDROP.dataURL
      • NgasDROP.generate_reproduce_data()
      • NgasDROP.getIO()
      • NgasDROP.initialize()
      • NgasDROP.setCompleted()
  • dlg.data.drops.rdbms
    • RDBMSDrop
      • RDBMSDrop.dataURL
      • RDBMSDrop.generate_reproduce_data()
      • RDBMSDrop.getIO()
      • RDBMSDrop.initialize()
      • RDBMSDrop.insert()
      • RDBMSDrop.select()
  • dlg.data.drops.json_drop
  • dlg.data.drops.s3_drop
• dlg.dropmake
  • dlg.dropmake.pg_generator
    • apply_config()
    • fill()
    • partition()
    • resource_map()
    • unroll()
  • dlg.dropmake.scheduler
    • DAGUtil
      • DAGUtil.build_dag_from_drops()
      • DAGUtil.ganttchart_matrix()
      • DAGUtil.get_longest_path()
      • DAGUtil.get_max_antichains()
      • DAGUtil.get_max_dop()
      • DAGUtil.get_max_width()
      • DAGUtil.label_schedule()
      • DAGUtil.prune_antichains()
    • KFamilyPartition
      • KFamilyPartition.add_node()
    • MinNumPartsScheduler
      • MinNumPartsScheduler.override_cannot_add()
    • MySarkarScheduler
      • MySarkarScheduler.override_cannot_add()
      • MySarkarScheduler.partition_dag()
      • MySarkarScheduler.reduce_partitions()
    • PSOScheduler
      • PSOScheduler.constrain_func()
      • PSOScheduler.objective_func()
      • PSOScheduler.partition_dag()
    • Partition
      • Partition.add()
      • Partition.add_node()
      • Partition.can_add()
      • Partition.probe_max_dop()
      • Partition.remove()
      • Partition.schedule
    • Schedule
      • Schedule.efficiency
      • Schedule.schedule_matrix
      • Schedule.workload
    • Scheduler
      • Scheduler.map_partitions()
      • Scheduler.merge_partitions()
    • SchedulerException
  • dlg.dropmake.pg_manager
    • PGManager
      • PGManager.add_pgt()
      • PGManager.get_gantt_chart()
      • PGManager.get_pgt()
      • PGManager.get_schedule_matrices()
    • PGUtil
      • PGUtil.vstack_mat()