NuGet Perf–Setup and Testing

i built a web api application to serve as the test bed. it has a ravencontroller, which looks like this:

public class ravencontroller : apicontroller
{
    private static idocumentstore documentstore;

    public static idocumentstore documentstore
    {
        get
        {
            if (documentstore == null)
            {
                lock (typeof (ravencontroller))
                {
                    if (documentstore != null)
                        return documentstore;
                    documentstore = new documentstore
                        {
                            url = "http://localhost:8080",
                            defaultdatabase = "nuget"
                        }.initialize();
                    indexcreation.createindexes(typeof (packages_search).assembly, documentstore);
                }
            }
            return documentstore;
        }
    }

    public idocumentsession documentsession { get; set; }

    public override async task<httpresponsemessage> executeasync(httpcontrollercontext controllercontext, cancellationtoken cancellationtoken)
    {
        using (documentsession = documentstore.opensession())
        {
            httpresponsemessage result = await base.executeasync(controllercontext, cancellationtoken);
            documentsession.savechanges();
            return result;
        }
    }
}

and now we have the following controllers:

public class packagescontroller : ravencontroller
{
    public ienumerable<packages_search.reduceresult> get(int page = 0)
    {
        return documentsession.query<packages_search.reduceresult, packages_search>()
            .where(x=>x.isprerelease == false)
            .orderbydescending(x=>x.downloadcount)
                .thenby(x=>x.created)
            .skip(page*30)
            .take(30)
            .tolist();
    }
}

public class searchcontroller : ravencontroller
{
    public ienumerable<packages_search.reduceresult> get(string q, int page = 0)
    {
        return documentsession.query<packages_search.reduceresult, packages_search>()
            .search(x => x.query, q)
            .where(x => x.isprerelease == false)
            .orderbydescending(x => x.downloadcount)
                .thenby(x => x.created)
            .skip(page * 30)
            .take(30)
            .tolist();
    }
}

and, just for the sake of completeness, the packages_search index looks like this:

public class packages_search : abstractindexcreationtask<package, packages_search.reduceresult>
{
    public class reduceresult
    {
        public datetime created { get; set; }
        public int downloadcount { get; set; }
        public string packageid { get; set; }
        public bool isprerelease { get; set; }
        public object[] query { get; set; }
    }

    public packages_search()
    {
        map = packages => from p in packages
                          select new
                              {
                                  p.created, 
                                  downloadcount = p.versiondownloadcount, 
                                  p.packageid, 
                                  p.isprerelease,
                                  query = new object[] { p.tags, p.title, p.packageid}
                              };
        reduce = results =>
                 from result in results
                 group result by new {result.packageid, result.isprerelease}
                 into g
                 select new
                         {
                             g.key.packageid,
                             g.key.isprerelease,
                             downloadcount = g.sum(x => x.downloadcount),
                             created = g.select(x => x.created).orderby(x => x).first(),
                             query = g.selectmany(x=>x.query).distinct()
                         };

        store(x=>x.query, fieldstorage.no);
    }
}

all right, that's enough setup -- now let's talk about the actual structure of the load tests. for these, we used vs 2012 load testing tool. we defined the following tests...

just browsing through the packages listing:

browsing a bit then searching, and then narrowing the search:

and finally, searching a few packages by their id, tags, etc:

i then defined the following load test:

with the following distribution:

finally, we have the way we actually run the test:

we get 20 seconds of warm up, then 5 minutes of tough load.