{ "id": "89730", "key": "TIMOB-8640", "fields": { "issuetype": { "id": "5", "description": "The sub-task of the issue", "name": "Sub-task", "subtask": true }, "parent": { "id": "89765", "key": "TIMOB-8652", "fields": { "summary": "Core: Create a new Titanium Command Line Interface", "status": { "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "name": "Closed", "id": "6", "statusCategory": { "id": 3, "key": "done", "colorName": "green", "name": "Done" } }, "priority": { "name": "High", "id": "2" }, "issuetype": { "id": "6", "description": "gh.issue.epic.desc", "name": "Epic", "subtask": false } } }, "project": { "id": "10153", "key": "TIMOB", "name": "Titanium SDK/CLI", "projectCategory": { "id": "10100", "description": "Titanium and related SDKs used in application development", "name": "Client" } }, "fixVersions": [ { "id": "13271", "description": "Release 2.1.0", "name": "Release 2.1.0", "archived": false, "released": true, "releaseDate": "2012-06-29" }, { "id": "13277", "name": "Sprint 2012-08", "archived": true, "released": true, "releaseDate": "2012-04-22" } ], "resolution": { "id": "1", "description": "A fix for this issue is checked into the tree and tested.", "name": "Fixed" }, "resolutiondate": "2012-04-13T12:35:13.000+0000", "created": "2012-04-09T17:16:48.000+0000", "priority": { "name": "Medium", "id": "3" }, "labels": [ "core" ], "versions": [], "issuelinks": [ { "id": "16564", "type": { "id": "10020", "name": "Depends", "inward": "is dependent of", "outward": "depends on" }, "inwardIssue": { "id": "89981", "key": "TIMOB-8718", "fields": { "summary": "Core: Compare possible CLI technologies", "status": { "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "name": "Closed", "id": "6", "statusCategory": { "id": 3, "key": "done", "colorName": "green", "name": "Done" } }, "priority": { "name": "Medium", "id": "3" }, "issuetype": { "id": "5", "description": "The sub-task of the issue", "name": "Sub-task", "subtask": true } } } } ], "assignee": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "updated": "2012-06-15T16:29:55.000+0000", "status": { "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "name": "Closed", "id": "6", "statusCategory": { "id": 3, "key": "done", "colorName": "green", "name": "Done" } }, "components": [ { "id": "10207", "name": "Tooling" } ], "description": "We need to investigate what it would take to migrate build infrastructure from python to node.js (or another alternative, ideally a solution that allows us to use JS end-to-end.) We need to figure out issues like:\r\n\r\n* Distribution (footprint, installation/distribution, etc.)\r\n* API assessment\r\n* Supporting infrastructure\r\n* IPC (for studio and drillbit/simulator integration)", "attachment": [], "flagged": false, "summary": "CLI: Evaluate using node.js in build scripts", "creator": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "subtasks": [], "reporter": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "environment": null, "closedSprints": [ { "id": 3, "state": "closed", "name": "Release 3.0.0", "startDate": "2012-09-27T05:26:46.636Z", "endDate": "2012-10-08T20:05:00.000Z", "completeDate": "2012-12-20T17:03:19.403Z" } ], "comment": { "comments": [ { "id": "190481", "author": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "body": "Here are the results of the initial investigation:\r\n\r\n\r\nh6. Footprint\r\n~7.5MB (base install, no npm packages, from brew on OS X)\r\n\r\nh6. Distribution:\r\nWindows: MSI, chocolatey\r\n* From the README on git: \"Windows builds are not yet satisfactorily stable.\"\r\n* Can distribute the same way we do python now (complete install 'encapsulated' in Studio or pulled and installed from MSI, which is called out to by a plugin)\r\n\r\nOS X: brew, port, pkg installer, source\r\nLinux: deb, apt, RPM, yum, pacman, source\r\n\r\nFor OS X and linux, either we would need brew, port, or a package manager (identified by command availability?) installed (to facilitate updating.) An advantage of doing such is that we could start bundling mobile releases (and modules?) as repos for package management tools (brew would be especially easy.) Another alternative is to simply lock a node version that gets distributed with Studio only, so that we can be guaranteed a consistent API, although this will take some work to periodically upgrade.\r\n\r\nRapid releases and unstable development make working with node (esp. on Windows) much more difficult; we'd need a way to autoupdate (does the MSI install or OS X PKG do this? package managers certainly can)\r\n\r\n\r\nh6. Supported required featuresets\r\n\r\n* Support natively:\r\n** Async sockets/HTTP\r\n** Child process fork with I/O redirect\r\n** 'assert' (can replace drillbit asserts on server-side, possibly)\r\n** Filesystem access / pathing tools\r\n\r\n* Support from npm:\r\n** mysql (Studio registry interaction)\r\n** xcode (!!! - but last 63 days ago, no repo listed on npm)\r\n** *NOTE:* No direct adb or other android support (internal project?)\r\n\r\n* Other robust npm support:\r\n** colors\r\n** commander (robust command-line invoke module)\r\n** underscore (ruby-esque features)\r\n\r\nh6. Advantages:\r\n\r\n* End-to-end javascript\r\n* CLI can serve as a drillbit driver and command-line debug interface\r\n* Makes it easier to get non-PE involved in build modifications (i.e. CS/CE can deliver custom build solutions with minimal extra JS training)\r\n* Because node acts as a server, this may give us some distributed build or other automation capabilities\r\n* Eclipse as node debug driver: https://github.com/joyent/node/wiki/Using-Eclipse-as-Node-Applications-Debugger\r\n\r\nh6. Alternatives:\r\n\r\nThe primary concern for Node at this point is good Windows support, but because Microsoft has selected node.js as part of the Azure Cloud infrastructure, it's likely that node will rapidly improve in this area (possibly becoming robust long before we're ready to release a new CLI.)\r\n\r\nBasically the only real alternative we have at this point is another robust scripting language, likely either Python or Ruby. The primary disadvantage for either of these is good async server support (which both have, as external modules/plugins) which we might want for automation, debugging, etc.\r\n\r\nThe other primary disadvantages would be the lack of end-to-end JS and additional training (or involvement of Core PE) for delivering custom build solutions that customers may require.", "updateAuthor": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "created": "2012-04-10T14:27:10.000+0000", "updated": "2012-05-08T13:20:47.000+0000" }, { "id": "190889", "author": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "body": "We recommend against node as the primary driver for the new CLI, due to the lack of robustness, rapid speed of the project, and existence of suitable alternatives.", "updateAuthor": { "name": "stephentramer", "key": "stephentramer", "displayName": "Stephen Tramer", "active": true, "timeZone": "America/Los_Angeles" }, "created": "2012-04-13T10:37:07.000+0000", "updated": "2012-04-13T12:34:45.000+0000" }, { "id": "193930", "author": { "name": "cbarber", "key": "cbarber", "displayName": "Chris Barber", "active": true, "timeZone": "America/Chicago" }, "body": "h1. Overview\r\n\r\nI, along with the help of Bryan Hughes, have performed our own technical analysis of node.js and found it to be viable solution for command line based tools.\r\n\r\nWe identified a number of key requirements, some of which are driven by the needs of Mobile Web.\r\n\r\n* Distributable\r\n* Synchronous and asynchronous\r\n* File I/O\r\n* File/Directory management\r\n* Subprocesses\r\n* Logging\r\n* Image resizing\r\n* JSON parsing/serializing\r\n* Text encoding\r\n* Templates\r\n* Zip file creation/extraction\r\n* Web server w/ proxy support\r\n* Package system w/ dependency support\r\n* XML parsing\r\n\r\nh2. Distributable\r\n\r\nWe are of the belief that any dependencies of Titanium Mobile should be distributed with or downloaded by Titanium Mobile. This includes the runtime that the CLI system depends on.\r\n\r\nnode.js would be easy to bundle with Titanium Mobile. It's a single executable binary file that clocks in at ~7MB on Mac OS X, ~10MB on Linux, and ~5MB on Windows. All libraries node.js uses are statically linked into the executable.\r\n\r\nIn addition to the node executable, node.js we would bundle npm for package management. npm is just under 3MB. npm is distributed as a node module which would be just one of the potentially many modules we may wish to distribute with Titanium Mobile.\r\n\r\nBoth node.js and npm are licensed under the MIT license and are compatible with Titanium Mobile's license.\r\n\r\nWe find that node.js to be suitable for distribution with Titanium Mobile.\r\n\r\nh2. Synchronous and Asynchronous\r\n\r\nBuild systems generally perform tasks serially, one at a time. Build times can be reduced by processing tasks in parallel. However a specific task is usually a series of synchronous tasks.\r\n\r\nMost of node.js's built-in file system functions have both synchronous and asynchronous versions. There are a handful of operations that are only asynchronous where we'd need a way to synchronize them.\r\n\r\nThere are three basic strategies for making asynchronous operations synchronous.\r\n\r\n# C++ addon\r\n# File system synchronization\r\n# Async task framework\r\n\r\nh3. C++ Addon\r\n\r\nnode.js supports C++ \"addons\" that plug directly into V8. If we need synchronous HTTP requests or subprocesses, we can custom build a C++ solution.\r\n\r\nPros:\r\n* We can do literally anything\r\n* C++ code is very fast\r\n\r\nCons:\r\n* Requires C++ experience\r\n* Modules are homebrewed and not battle tested\r\n* We would need to distribute binaries for each platform\r\n\r\nh3. File system synchronization\r\n\r\nThere is a hack where we can block node.js until an asynchronous process completes.\r\n\r\nThe hack consists of the following:\r\n\r\n* Synchronously write a small file containing JavaScript code that performs the async only function\r\n* Asynchronously execute another node process that runs that small JavaScript file\r\n* In a while(true) loop, synchronously read a temp file that marks the completion of the asynchronous node subprocess\r\n\r\nThe synchronous file access forces node.js to context switch allowing the kernel to schedule time to the node subprocess.\r\n\r\nPros:\r\n* Pure JavaScript solution\r\n* Synchronous subprocess implementation already exists in shell.js module\r\n\r\nCons:\r\n* Requires at least two temp files\r\n* Continuous synchronous file reads in while loop may affect performance of subprocess\r\n\r\nh3. Async Task Framework\r\n\r\nPerhaps the best strategy is to go with the flow of node.js. If node is async, then a framework that embraces node's asynchronous concepts would allow us to adapt to both sync and async tasks.\r\n\r\nThis is not a new concept. In fact, it closely resembles the same process in Mobile Web's AMD loader. Bryan has built a prototype to demonstrate a solution.\r\n\r\nThe idea is to define multiple tasks that would run asynchronously. Tasks can depend on the completion of other tasks. When a task completes, it simply fires the finished() or error() callback.\r\n\r\n~~~\r\nvar fs = require(\"fs\"),\r\n\tTask = require(\"task\"),\r\n t = new Task;\r\n\r\nt.add(\"resize images\", function(finished, error) {\r\n\t// resize some images\r\n\tfinished();\r\n});\r\n\r\nt.add(\"copy images\", [\"resize images\"], function(finished, error) {\r\n\t// copy some images\r\n\tcp(\"source\", \"dest\") ? finished() : error();\r\n});\r\n\r\nt.run();\r\n~~~\r\n\r\nPros:\r\n* Pure JavaScript solution\r\n* Gracefully handles async and sync functions\r\n\r\nCons:\r\n* Just requires a little different thinking\r\n\r\nh3. Synchronous and Asynchronous Conclusion\r\n\r\nAfter comparing the 3 solutions, we recommend using a async task framework, namely the one Bryan has started building. Only in special situations should we resort to C++ addons. File system synchronization is a hack and should only be considered as a fallback for a C++ addon.\r\n\r\nWe find that node.js to be suitable for both synchronous and asynchronous tasks with a little help from a module/addon.\r\n\r\nh2. Package system w/ dependency support\r\n\r\nWe need the ability to bundle up files, binaries, documentation, and examples into distributable packages. The npm (node.js package manager) provides a means to installing packages and their dependencies.\r\n\r\nPackages can contain multiple modules and C++ addons. In the top-level directory of the package is a manifest file called \"package.json\". This file describes various meta data for the package including dependencies.\r\n\r\nWhen a package is downloaded, it's dependencies are fetched and installed.\r\n\r\nnpm is flexible and allows us to host our own packages. Packages can be installed a couple of ways including by specifying a URL to a tarball or from a git repository.\r\n\r\nWe find that package with dependency support via npm to be suitable for Titanium Mobile.\r\n\r\nh2. File I/O\r\n\r\nnode.js has a file system module containing various functions for manipulating files.\r\n\r\nThe easiest interface for reading files is the readFile() and readFileSync() functions. Similarly, writing files is writeFile() and writeFileSync().\r\n\r\n~~~\r\nvar fs = require(\"fs\");\r\nvar passwd = fs.readFileSync(\"/etc/passwd\");\r\n~~~\r\n\r\nThe file system module also offers generic read() and write() functions for writing to any sort of file descriptor.\r\n\r\nWe find that the file system functions for file I/O to be suitable for Titanium Mobile.\r\n\r\nh2. File/Directory Manipulation\r\n\r\nThe file system module also provides functions for virtually every file or directory operation Titanium Mobile would need such as creating, listing, copying, and deleting directories.\r\n\r\n~~~\r\nvar fs = require(\"fs\");\r\nfs.readdirSync(\"/etc\").forEach(function(file) {\r\n\tconsole.log(file);\r\n});\r\n\r\nfs.mkdirSync(\"/tmp/titanium\");\r\nfs.rmdirSync(\"/tmp/titanium\");\r\n~~~\r\n\r\nMoving a file or directory will require a helper function that would copy a source to a destination, then delete the source.\r\n\r\nThere is an excellent node module called [shell.js|https://github.com/arturadib/shelljs] that defines a number of functions that mimic Unix command names including cd(), ls(), mkdir(), pwd(). One interesting function is exec() for which support synchronous subprocesses.\r\n\r\nWe find that the file system functions for file manipulation to be suitable for Titanium Mobile.\r\n\r\nh2. Subprocesses\r\n\r\nSubprocesses are either started using node's exec() or spawn() functions. Unfortunately, these are both asynchronous. To make these synchronous, we would need to use one of the methods described in the Synchronous and Asynchronous section.\r\n\r\nexec() is capable of running a subprocess and when the subprocess completes, a callback is fired with the stdout and stderr. exec() doesn't support interactive execution and thus has no way of passing data to the subprocess over stdin.\r\n\r\nspawn() on the other hand is capable of passing data over stdin to the subprocess as well as receiving data from stdout and stderr while the subprocess is running.\r\n\r\nBased on a quick survey of all subprocess calls in the current Titanium Mobile build scripts, it appears that exec() would be sufficient for nearly all scenarios.\r\n\r\nWe find that subprocess support to be suitable (and in some cases with the help of a module/addon) for Titanium Mobile.\r\n\r\nh2. JSON parsing/serializing\r\n\r\nnode.js has built-in support for both parsing and serializing JSON data.\r\n\r\n~~~\r\nvar data = { a:1 };\r\nvar str = JSON.stringify(data);\r\nconsole.log(JSON.parse(str));\r\n~~~\r\n\r\nWe find that JSON support in node.js to be suitable for Titanium Mobile.\r\n\r\nh2. Logging\r\n\r\nnode.js has built-in functions logging messages to the terminal (console). The console functions support automatically stringifying objects and arrays.\r\n\r\nWhile log() and info() calls output to stdout, warn() and error() output to stderr.\r\n\r\nA more sophisticated logger that could ignore log messages below a specific log level is desired. Additionally, it might be beneficial to support logging to various targets (both file system and network based).\r\n\r\nSince node.js's logging mechanisms aren't capable of doing sophisticated logging, we either have to use one of the two dozen loggers in npm or write our own.\r\n\r\nWriting our own logger is trivial. Loggers in npm should be evaluated and if there isn't a perfect solution, we can use it as a starting point for our own logger.\r\n\r\nIt's worth mentioning that it's possible to render color text in the terminal which might be appealing for the Titanium Mobile CLI.\r\n\r\nWe find that there are sufficient logging modules or a custom logger can be written that would be suitable for Titanium Mobile.\r\n\r\nh2. Image resizing\r\n\r\nnode.js does not have any built-in functions to resize images, however there is an ImageMagick module that we can use.\r\n\r\nSomewhat related, there is a node.js module called [node-png|https://github.com/pkrumins/node-png] that we can use to optimize PNG images.\r\n\r\nWe find that are suitable image resizing solutions for Titanium Mobile.\r\n\r\nh2. Text encoding\r\n\r\nnode.js supports \"ascii\", \"utf8\", \"ucs2\", \"base64\", \"binary\", and \"hex\" encodings via the built-in Buffer object.\r\n\r\nnode.js does not have support for recoding ISO-8859-1 text, though there is a module that wraps iconv that does (*nix only). The algorithm to convert ISO-8859-1 text to UTF-8 and back should be relatively trivial to implement.\r\n\r\nSince the vast majority of the text and code will be in UTF-8, we find node.js to be suitable for encoding text for Titanium Mobile.\r\n\r\nh2. Templates\r\n\r\nThere are number of places where we need a template engine that will do simple variable substitution as well as scrub text for HTML entities and trim strings.\r\n\r\nWhile node.js doesn't have a built-in template engine, there are several in npm that would work.\r\n\r\nA module such as [handlebars.js|https://github.com/wycats/handlebars.js] would get the job done.\r\n\r\nWe find that there are many viable template engine modules that would be suitable for Titanium Mobile.\r\n\r\nh2. Web server w/ proxy support\r\n\r\nnode.js ships with an HTTP server that is capable of both http and https connections. Furthermore, there are a number of web server modules for node.js that implement missing features such as 404 handling and mime types.\r\n\r\nSetting up a proxy server on top of node.js is trivial.\r\n\r\nOne thing that is not easy to support is video streaming. In some web browsers, they expect the video content to be chunked encoded and sent over the wire as HTTP 206 partial content chunks.\r\n\r\nnode.js also has a module called [Socket.IO|http://socket.io/] that can be used to build server pushed messages to the web browser. This feature will be key for debug tools needed for Mobile Web.\r\n\r\nWe find that web server including proxy support in node.js to be suitable for Titanium Mobile.\r\n\r\nh2. Zip file creation/extraction\r\n\r\nnode.js has built-in gzip support. This is not the same as zip file support, but is used to compress the individual files in a zip archive.\r\n\r\nUsing the [adm-zip|https://github.com/cthackers/adm-zip] module, we can support creating and extracting zip files. This module can list all the entries in the zip file, update the zip file, and more.\r\n\r\nIt is not known at this time if adm-zip is capable of producing Android APK-compatible zip files. There are other zip file modules available that might be able to support Android APK-compatible zip files.\r\n\r\nAside from not knowing precisely if APK files are supported, we find that zip file support via a module is suitable for Titanium Mobile.\r\n\r\nh2. XML parsing\r\n\r\nnode.js does not have any built-in support for parsing XML, however there are a number of modules that can parse using either a SAX or DOM parser.\r\n\r\nThe [node-o3-xml|https://github.com/ajaxorg/node-o3-xml] module also includes support for XPath and namespaces. Note: node-03-xml only works on Windows via Cygwin, so probably not a great solution.\r\n\r\nSeveral XML parsers leverage libxml2 which will most likely be a platform specific dependency we would need to manage.\r\n\r\nThere are some XML parsers that are pure JavaScript implementations that would work for parsing files such as the tiapp.xml or timodule.xml files.\r\n\r\nWe find that there are viable XML parsing solutions suitable for Titanium Mobile.\r\n\r\nh1. Conclusion\r\n\r\nnode.js proves to be a very suitable candidate for use in Titanium Mobile. In general, here are a few pros and cons:\r\n\r\nPros:\r\n* Active and healthy community\r\n* JavaScript language, something we all know\r\n* Very fast\r\n* Easily distributable\r\n* Support for native C++ addons\r\n* Fast paced development cycle\r\n\r\nCons:\r\n* Asynchronous programming requires different thinking\r\n* Compared to other alternatives, node is much younger\r\n* Fast paced development cycle\r\n\r\nI have not found any evidence that node.js is any more or less unstable or not robust than any other alternative.\r\n\r\nBased on our experience and research, we have concluded that node.js is a viable solution for the CLI and build tools. Regardless of the selected technology choice, we plan to use node.js for all Mobile Web's build related processes, server-side reference implementations, and possibly AST parsing.", "updateAuthor": { "name": "cbarber", "key": "cbarber", "displayName": "Chris Barber", "active": true, "timeZone": "America/Chicago" }, "created": "2012-05-07T03:53:28.000+0000", "updated": "2012-05-07T03:53:28.000+0000" }, { "id": "193969", "author": { "name": "bhughes", "key": "bhughes", "displayName": "Bryan Hughes", "active": true, "timeZone": "America/Los_Angeles" }, "body": "I have put my async framework up on github at https://github.com/bryan-m-hughes/node-taskman. It isn't complete yet, but should be soon.", "updateAuthor": { "name": "bhughes", "key": "bhughes", "displayName": "Bryan Hughes", "active": true, "timeZone": "America/Los_Angeles" }, "created": "2012-05-07T10:17:12.000+0000", "updated": "2012-05-07T10:17:12.000+0000" }, { "id": "193972", "author": { "name": "bhughes", "key": "bhughes", "displayName": "Bryan Hughes", "active": true, "timeZone": "America/Los_Angeles" }, "body": "Regarding zip files: APK compliant zip files is not a requirement because the Android CLI tools can create them for you. Since APK compliant zip files are only required for Android, and since you can't build an Android app without the SDK installed, this support is guaranteed to always be available. Read http://developer.android.com/guide/developing/building/building-cmdline.html for more information.", "updateAuthor": { "name": "bhughes", "key": "bhughes", "displayName": "Bryan Hughes", "active": true, "timeZone": "America/Los_Angeles" }, "created": "2012-05-07T10:28:21.000+0000", "updated": "2012-05-07T10:28:21.000+0000" } ], "maxResults": 9, "total": 9, "startAt": 0 } } }