SCons integrationΒΆ

This SConstruct file is an example of using nestly with the SCons build system:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
# -*- python -*-
#
# This example takes every file in the inputs directory and performs the
# following operations:
# * cuts out a column range from every line in the file; either 1-5 or 3-40
# * optionally filters out every line that has an "o" or "O"
# * runs wc on every such file
# * aggregate these together using the prep_tab.sh script
#
# Assuming that SCons is installed, you should be able to run this example by
# typing `scons` in this directory. That should build a series of things in the
# `build` directory. Because this is a build system, deleting a file or directory
# in the build directory and then running scons will simply rerun the needed parts.

from os.path import join
import os

from nestly.scons import SConsWrap
from nestly import Nest

nest = SConsWrap(Nest(), 'build')


# Add our aggregate targets, initializing collections that will get populated
# downstream. At the end of the pipeline, we will operate on these collections.
# The `add_argument` takes a key which will be the key used for accessing the
# collection. The `list` argument specifies that the collection will be a list.
nest.add_aggregate('count_agg', list)
nest.add_aggregate('cut_agg', list)

# Add a nest level with the name 'input_file' that takes the files in the inputs
# directory as its nestable list. Make its label function just the basename.
nest.add('input_file', [join('inputs', f) for f in os.listdir('inputs')],
      label_func=os.path.basename)

# This nest level determines the column range we will cut out of the file.
nest.add('cut_range', ['1-5', '3-40'])

# This adds a nest item with the name 'cut' and makes an SCons target out of
# the result.
@nest.add_target()
def cut(outdir, c):
    cut, = Command(join(outdir, 'cut'),
                   c['input_file'],
                   'cut -c {0[cut_range]} <$SOURCE >$TARGET'.format(c))
    # Here we add this cut file to the all_cut aggregator before returning
    c['cut_agg'].append(cut)
    return cut

# This nest level determines whether we remove the lines with o's.
nest.add('o_choice', ['remove_o', 'leave_o'])

@nest.add_target()
def o_choice(outdir, c):
    # If we leave the o lines, then we don't have to do anything.
    if c['o_choice'] == 'leave_o':
        return c['cut']
    # If we want to remove the o lines, then we have to make an SCons Command
    # that does so with sed.
    return Command(join(outdir, 'o_removed'),
                   c['cut'],
                   'sed "/[oO]/d" <$SOURCE >$TARGET')[0]

# Add a target for the word counts.
@nest.add_target()
def counts(outdir, c):
    counts, = Command(join(outdir, 'counts'),
                      c['o_choice'],
                      'wc <$SOURCE >$TARGET')
    # Add the resulting file to the count_agg collection
    c['count_agg'].append(counts)
    return counts

# Add a control dictionary with chosen values to each leaf directory
nest.add_controls(Environment())

# Before operating on our aggregate collections, we return back to the original
# nest level in which the aggregates were created by using the `pop` function to
# remove all of the later nest levels from the nest state, leaving only the
# collections.
nest.pop('input_file')

# Now, back at the initial nest level, we can operate on the populated aggregate
# collections. First, the counts:
@nest.add_target()
def all_counts(outdir, c):
    return Command(join(outdir, 'all_counts.tab'),
                   c['count_agg'],
                   './prep_tab.sh $SOURCES | column -t >$TARGET')

# Then the cuts:
@nest.add_target()
def all_cut(outdir, c):
    return Command(join(outdir, 'all_cut.txt'),
                   c['cut_agg'],
                   'cat $SOURCES >$TARGET')