JS Build Object Recursively

I am attempting to build a file-structure index using nodeJS. I'm using the fs.readir function to iterate the files, which works fine. My problem is descending into the directory s

Solution 1:

The result is only available asynchronously, so you are trying to output the result too soon. The inner code is only executed later.

You can solve this in many ways. A very nice solution to working with asynchronous code is using promises.

As you have a recursive call, you'll have to resolve that with promises too.

NB: Note you had a bug in the comparison with "dir": you assigned instead of comparing.

Here is how your code would look:

var indexer = function(directory) {
    // return a promise object
    return new Promise(function (resolve, reject) {
        Self.indexleft++;
        var results = {};
        Self.client.readdir(directory, function(err,fileLst){
            if(err) { 
                reject(); // promise is rejected
                return;
            }
            // "Iterate" over file list asyonchronously
            (function nextFile(fileList) {
                if (!fileList.length) {
                    resolve(results);  // promise is resolved
                    return;
                }
                var file = fileLst.shift(); // shop off first file
                var ident = identify(file); 
                results[ident.name] = ident;
                if(ident.type === 'dir'){ // There was a bug here: equal sign!
                    var descendant = directory !== '' 
                            ? directory + '\\' + ident.name : ident.name;
                    // recursively call indexer: it is again a promise!        
                    indexer(descendant).then(function (result) {
                        ident.children = result;
                        // recursively continue with next file from list
                        nextFile(fileList);
                    });
                } else {
                    nextFile(fileLst);
                }
            })(fileLst); // start first iteration with full list
        });
    });
};

// Call as a promise. Result is passed async to callback. 
indexer('').then(function(me) {
    console.log(me);
});

I made some dummy functions for your external references to make this snippet work:

// Below code added to mimic the external references -- can be ignored
var filesystem = [
    "",
    "images",
    "images\\photo.png",
    "images\\backup",
    "images\\backup\\old_photo.png",
    "images\\backup\\removed_pic.jpg",
    "images\\panorama.jpg",
    "docs",
    "docs\\essay.doc",
    "readme.txt",
];

var Self = {
    indexLeft: 0,
    client: {
        readdir: function (directory, callback) {
            var list = filesystem.filter( path => 
                    path.indexOf(directory) == 0 
                    && path.split('\\').length == directory.split('\\').length + (directory!=='')
                    && path !== directory
            ).map ( path => path.split('\\').pop() );
            setTimeout(callback.bind(null, 0, list), 100);
        }
    }
}

function identify(item) {
    return {
        name: item,
        type: item.indexOf('.') > -1 ? 'file' : 'dir'
    };
}
// Above code added to mimic the external references -- can be ignored

var indexer = function(directory) {
    // return a promise object
    return new Promise(function (resolve, reject) {
        Self.indexleft++;
        var results = {};
        Self.client.readdir(directory, function(err,fileLst){
            if(err) { 
                reject(); // promise is rejected
                return;
            }
            // "Iterate" over file list asyonchronously
            (function nextFile(fileList) {
                if (!fileList.length) {
                    resolve(results);  // promise is resolved
                    return;
                }
                var file = fileLst.shift(); // shop off first file
                var ident = identify(file); 
                results[ident.name] = ident;
                if(ident.type === 'dir'){ // There was a bug here: equal sign!
                    var descendant = directory !== '' 
                            ? directory + '\\' + ident.name : ident.name;
                    // recursively call indexer: it is again a promise!        
                    indexer(descendant).then(function (result) {
                        ident.children = result;
                        // recursively continue with next file from list
                        nextFile(fileList);
                    });
                } else {
                    nextFile(fileLst);
                }
            })(fileLst); // start first iteration with full list
        });
    });
};

// Call as a promise. Result is passed async to callback. 
indexer('').then(function(me) {
    console.log(me);
});

Solution 2:

It's not really obvious how you're expecting to that returned object from the code you have, but I can help you get the object nonetheless.

The shape of the object is bad because you're using filenames as keys on the object but that's wrong. Keys should be identifiers known to your program, and since filenames can be almost anything, using a filename as a key is terrible.

For example, consider if a file was named name in your structure

{ "Applications" : {
    "name" : "Applications",
    "type" : "dir",
    "name" : {
      "name" : "name"
       ... } } }

Yep, it just broke. Don't worry tho, our solution won't run into such troubles.

const co = require('co')
const {stat,readdir} = require('fs')
const {extname,join} = require('path')

// "promisified" fs functions
const readdirp = path =>
  new Promise ((t,f) => readdir (path, (err, res) => err ? f (err) : t (res)))

const statp = fd =>
  new Promise ((t,f) => stat (fd, (err,stats) => err ? f (err) : t (stats)))

// tree data constructors
const Dir = (path, children) =>
  ({type: 'd', path, children})

const File = (path, ext) =>
  ({type: 'f', path, ext})

// your function
const indexer = function* (path) {
  const stats = yield statp (path)
  if (stats.isDirectory ())
    return Dir (path, yield (yield readdirp (path)) .map (p => indexer (join (path,p))))
  else
    return File (path, extname (path))
}

This is good design because we didn't tangle directory tree building in with whatever Self.client is. Parsing a directory and building a tree is its own thing, and if you need an Object to inherit that behaviour there are other ways to do it.

Ok let's setup a sample tree of files and then run it

$ mkdir test
$ cd test
$ mkdir foo
$ touch foo/disk.iso foo/image.jpg foo/readme.txt
$ mkdir foo/bar
$ touch foo/bar/build foo/bar/code.js foo/bar/migrate.sql

Using indexer is easy

// co returns a Promise
// once indexer is done, you will have a fully built tree
co (indexer ('./test')) .then (
  tree => console.log (JSON.stringify (tree, null, '  ')),
  err  => console.error (err.message)
)

Output (some \n removed for brevity)

{
  "type": "d",
  "path": "./foo",
  "children": [
    {
      "type": "d",
      "path": "foo/bar",
      "children": [
        { "type": "f", "path": "foo/bar/build", "ext": "" },
        { "type": "f", "path": "foo/bar/code.js", "ext": ".js" },
        { "type": "f", "path": "foo/bar/migrate.sql", "ext": ".sql" }
      ]
    },
    { "type": "f", "path": "foo/disk.iso", "ext": ".iso" },
    { "type": "f", "path": "foo/image.jpg", "ext": ".jpg" },
    { "type": "f", "path": "foo/readme.txt", "ext": ".txt" }
  ]
}

---

If you try indexer on a path to a file, it will not fail

co (indexer ('./test/foo/disk.iso')) .then (
  tree => console.log (JSON.stringify (tree, null, '  ')),
  err  => console.error (err.message)
)

Output

{ "type": "f", "path": "./foo/disk.iso", "ext": ".iso" }