Unit testing MongoDB queries

When writing unit tests, it’s common to stop at the database access layer.
You might have a “dumb” data access layer that just passes stored procedure names and parameters to the SQL database, for instance.
It’s usually very hard to test this layer, since it requires either that your build server has access to a SQL Server instance, or that a SQL Server is running on the build server.
Plus, we’re getting out of unit tests and are entering integration tests, here.

Using a more advanced tool like Entity Framework, in order to test your complex EF queries, there are usually methods to insert fake data into a fake container, like Test Doubles, InMemory, or Effort.

Using MongoDB, you might encounter the same problem : how do I test that my complex queries are working ?

Here for instance, I get the possible colors of a product; how do I know it works using unit tests?

public IEnumerable<Color> GetColors(string productId) {
	var coll = db.GetCollection<BsonDocument>("products");
	var aggregate = coll.Aggregate(new AggregateOptions { AllowDiskUse = true })
		.Match(new BsonDocument { { "ProductId", productId }, { "Color", new BsonDocument("$ne", "") } })
		.Unwind(new StringFieldDefinition<BsonDocument>("Color"))
		.Group(new BsonDocument { { "_id", "$Color" }, { "Count", new BsonDocument("$sum", 1) } })
		.Sort(Builders<BsonDocument>.Sort.Descending("Count"))
		.Limit(100);

	return aggregate
		.ToList()
		.Select(doc => new Color { ColorValue = doc["_id"].AsString, Count = doc["Count"].AsInt32 });
}

public IEnumerable<Color> GetColors(string productId) {

var coll = db.GetCollection<BsonDocument>("products");

var aggregate = coll.Aggregate(new AggregateOptions { AllowDiskUse = true })

.Match(new BsonDocument { { "ProductId", productId }, { "Color", new BsonDocument("$ne", "") } })

.Unwind(new StringFieldDefinition<BsonDocument>("Color"))

.Group(new BsonDocument { { "_id", "$Color" }, { "Count", new BsonDocument("$sum", 1) } })

.Sort(Builders<BsonDocument>.Sort.Descending("Count"))

.Limit(100);

return aggregate

.ToList()

.Select(doc => new Color { ColorValue = doc["_id"].AsString, Count = doc["Count"].AsInt32 });

}

MongoDB has an “inMemory” storage engine, but it’s reserved to the (paid) Enterprise edition. Fortunately, since 3.2, even the Community edition has a not-very-well-documented “ephemeralForTests” storage engine, which loads up an in-memory Mongo instance, and does not store anything on the hard drive (but has poor performances). Exactly what we need!

Before running the data access layer tests, we will need to fire up an in-memory instance of MongoDB.
This instance will be common to all the tests for the layer, otherwise the test runner will fire up new tests faster than the system releases resources (file and ports locks).

You will have to extract the MongoDB binaries in your sources repository somewhere, and copy them besides your binaries on build.

The following wrapper provides a “Query” method that allows us to access the Mongo instance through command-line, bypassing the data access layer, in order to insert test data or query insertion results.

public class MongoDaemon : IDisposable {
	public const string ConnexionString = "mongodb://localhost:27017";
	public const string DbName = "test";
	public const string Host = "localhost";
	public const string Port = "27017";
	private readonly string assemblyFolder;
	private readonly string dbFolder;
	private readonly string mongoFolder;
	private Process process;

	public MongoDaemon() {
		this.assemblyFolder = Path.GetDirectoryName(new Uri(typeof(MongoDaemon).Assembly.CodeBase).LocalPath);
		this.mongoFolder = Path.Combine(this.assemblyFolder, "mongo");
		this.dbFolder = Path.Combine(this.mongoFolder, "temp");

		// re-create db folder if it exists
		if (Directory.Exists(this.dbFolder)) {
			Directory.Delete(this.dbFolder, true);
			Directory.CreateDirectory(this.dbFolder);
		}

		this.process = new Process();
		process.StartInfo.FileName = Path.Combine(this.mongoFolder, "mongod.exe");
		process.StartInfo.Arguments = "--dbpath " + this.dbFolder + " --storageEngine ephemeralForTest";
		process.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;
		process.Start();
	}

	public void Dispose() {
		Dispose(true);
		GC.SuppressFinalize(this);
	}

	public string Query(string query) {
		var output = string.Empty;

		var procQuery = new Process {
			StartInfo = new ProcessStartInfo {
				FileName = Path.Combine(this.mongoFolder, "mongo.exe"),
				Arguments = string.Format("--host {0} --port {1} --quiet --eval \"{2}\"", Host, Port, query),
				UseShellExecute = false,
				RedirectStandardOutput = true,
				CreateNoWindow = true
			},
		};

		procQuery.Start();

		// read query output
		while (!procQuery.StandardOutput.EndOfStream) {
			output += procQuery.StandardOutput.ReadLine() + "\n";
		}

		// wait 2 seconds max before killing it
		if (!procQuery.WaitForExit(2000)) {
			procQuery.Kill();
		}

		return output;
	}

	protected virtual void Dispose(bool disposing) {
		if (disposing) {
			// dispose managed resources
		}

		if (process != null && !process.HasExited) {
			process.Kill();
		}
	}
}

public class MongoDaemon : IDisposable {

public const string ConnexionString = "mongodb://localhost:27017";

public const string DbName = "test";

public const string Host = "localhost";

public const string Port = "27017";

private readonly string assemblyFolder;

private readonly string dbFolder;

private readonly string mongoFolder;

private Process process;

public MongoDaemon() {

this.assemblyFolder = Path.GetDirectoryName(new Uri(typeof(MongoDaemon).Assembly.CodeBase).LocalPath);

this.mongoFolder = Path.Combine(this.assemblyFolder, "mongo");

this.dbFolder = Path.Combine(this.mongoFolder, "temp");

// re-create db folder if it exists

if (Directory.Exists(this.dbFolder)) {

Directory.Delete(this.dbFolder, true);

Directory.CreateDirectory(this.dbFolder);

}

this.process = new Process();

process.StartInfo.FileName = Path.Combine(this.mongoFolder, "mongod.exe");

process.StartInfo.Arguments = "--dbpath " + this.dbFolder + " --storageEngine ephemeralForTest";

process.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;

process.Start();

}

public void Dispose() {

Dispose(true);

GC.SuppressFinalize(this);

}

public string Query(string query) {

var output = string.Empty;

var procQuery = new Process {

StartInfo = new ProcessStartInfo {

FileName = Path.Combine(this.mongoFolder, "mongo.exe"),

Arguments = string.Format("--host {0} --port {1} --quiet --eval \"{2}\"", Host, Port, query),

UseShellExecute = false,

RedirectStandardOutput = true,

CreateNoWindow = true

};

procQuery.Start();

// read query output

while (!procQuery.StandardOutput.EndOfStream) {

output += procQuery.StandardOutput.ReadLine() + "\n";

}

// wait 2 seconds max before killing it

if (!procQuery.WaitForExit(2000)) {

procQuery.Kill();

}

return output;

}

protected virtual void Dispose(bool disposing) {

if (disposing) {

// dispose managed resources

}

if (process != null && !process.HasExited) {

process.Kill();

}

We’re using the IClassFixture interface of xUnit to fire up a MongoDaemon instance that will be common to all our tests using it.
It means we need to clean up previously inserted test data at each run.

public class MongoDataStoreTests : IClassFixture<MongoDaemon> {
	private readonly MongoDaemon daemon;
	private readonly DataStore sut;

	public MongoDataStoreTests(MongoDaemon daemon) {
		this.daemon = daemon;
		this.sut = new DataStore("localhost", 27017);

		// cleanup previous tests data
		this.daemon.Query("db.products.drop()");
	}

	[Fact]
	public void Colors_are_counted() {
		// arrange : insert test data
		var id = "test";
		var nb = 5;
		for (int i = 0; i < nb; i++) {
			var color = string.Format("{0}{0}{0}", i, i, i);
			var data = "{ ProductId: '" + id + "', Color: '" + color + "' }";
			this.daemon.Query("db.products.insertOne(" + data + ")");
		}

		// act
		var result = this.sut.GetColors(id);

		// assert
		result.Should().HaveCount(nb);
	}
}

public class MongoDataStoreTests : IClassFixture<MongoDaemon> {

private readonly MongoDaemon daemon;

private readonly DataStore sut;

public MongoDataStoreTests(MongoDaemon daemon) {

this.daemon = daemon;

this.sut = new DataStore("localhost", 27017);

// cleanup previous tests data

this.daemon.Query("db.products.drop()");

}

[Fact]

public void Colors_are_counted() {

// arrange : insert test data

var id = "test";

var nb = 5;

for (int i = 0; i < nb; i++) {

var color = string.Format("{0}{0}{0}", i, i, i);

var data = "{ ProductId: '" + id + "', Color: '" + color + "' }";

this.daemon.Query("db.products.insertOne(" + data + ")");

}

// act

var result = this.sut.GetColors(id);

// assert

result.Should().HaveCount(nb);

}

There you have it: a kind-of-easy way to test your Mongo data access layer.

Mocking static methods

I’m working on a pretty old application with no unit tests whatsoever, and I’m trying to write some as I add features and fix bugs.

After successively finding out about MS Fakes, then that it’s only included in VS2012+ Premium/Ultimate and very hard to include in a continuous integration chain, I was pretty stumped until I found this, which works incredibly well.

Another option to transform the static method into a static Func or Action. For instance.

Original code:

C#

class Math { public static int Add(int x, int y) { return x + y; } }

1
2
3
4
5
6
7
8

class Math
{
    public static int Add(int x, int y)
    {
        return x + y;
    }
}

You want to “mock” the Add method, but you can’t. Change the above code to this:

C#

public static Func<int, int, int> Add = (x, y) => { return x + y; };

1
2
3
4
5

public static Func<int, int, int> Add = (x, y) =>
{
    return x + y;
};

Existing client code doesn’t have to change (maybe recompile), but source stays the same.

Now, from the unit-test, to change the behavior of the method, just reassign an in-line function to it:

C#

[TestMethod] public static void MyTest() { Math.Add = (x, y) => { return 11; }; }

1
2
3
4
5
6
7
8
9

[TestMethod]
public static void MyTest()
{
    Math.Add = (x, y) =>
    {
        return 11;
    };
}

Put whatever logic you want in the method, or just return some hard-coded value, depending on what you’re trying to do.

This may not necessarily be something you can do each time, but in practice, I found this technique works just fine.

Unit testing custom querystring-based authorization on a WebApi controller

I needed to add a very simple authorization mechanism to my API: use a query string parameter “api_key”, so that it’s compatible with Swagger (using Swashbuckle, there is a field “api_key” in the Swagger UI) and is easily callable through Ruby On Rails.

Following and adapting a nice tutorial, I have done the following.

Implement the authorization filter

Create an interface for your API key “getter”:

public interface IApiKeyProvider
{
    string GetApiKey();
}

public interface IApiKeyProvider

{

string GetApiKey();

}

Implement this interface; here it’s extremely simple:

public class WebConfigApiKeyProvider : IApiKeyProvider
{
    public string GetApiKey()
    {
        return ConfigurationManager.AppSettings["api_key"];
    }
}

public class WebConfigApiKeyProvider : IApiKeyProvider

{

public string GetApiKey()

{

return ConfigurationManager.AppSettings["api_key"];

}

Inject this interface through your dependency injector of choice. You don’t have to modify your controllers, which is great.

Then create a filter attribute to use this implementation:

public class ApiKeyAuthorizeAttribute : ActionFilterAttribute
{
    private const string apiKeyParamName = "api_key";

    public override void OnActionExecuting(HttpActionContext filterContext)
    {
        // get the IApiKeyProvider instance that has been injected (oh MVC4 that provides a dependency resolver, I love you)
        var provider = filterContext.ControllerContext.Configuration.DependencyResolver.GetService(typeof(IApiKeyProvider)) as IApiKeyProvider;
        // get the query string parameter ;
        // it must be parsed from the full query, since it's not actually part of the action, but an additional parameter
        var queryStringCollection = HttpUtility.ParseQueryString(filterContext.Request.RequestUri.Query);
        string apiKey = queryStringCollection[apiKeyParamName];

        if (string.IsNullOrEmpty(apiKey) || apiKey != provider.GetApiKey())
        {
            throw new HttpResponseException(HttpStatusCode.Unauthorized);
        }

        base.OnActionExecuting(filterContext);
    }
}

public class ApiKeyAuthorizeAttribute : ActionFilterAttribute

{

private const string apiKeyParamName = "api_key";

public override void OnActionExecuting(HttpActionContext filterContext)

{

// get the IApiKeyProvider instance that has been injected (oh MVC4 that provides a dependency resolver, I love you)

var provider = filterContext.ControllerContext.Configuration.DependencyResolver.GetService(typeof(IApiKeyProvider)) as IApiKeyProvider;

// get the query string parameter ;

// it must be parsed from the full query, since it's not actually part of the action, but an additional parameter

var queryStringCollection = HttpUtility.ParseQueryString(filterContext.Request.RequestUri.Query);

string apiKey = queryStringCollection[apiKeyParamName];

if (string.IsNullOrEmpty(apiKey) || apiKey != provider.GetApiKey())

{

throw new HttpResponseException(HttpStatusCode.Unauthorized);

}

base.OnActionExecuting(filterContext);

}

Now you just have to add the [ApiKeyAuthorize] attribute to your controller(s), and you now need to add the proper api_key query string parameter to all your requests.

Test the filter

A few things to test: that your class uses this attribute, and that the attribute does what it says it does.

Test the attribute presence

Here I’m using XUnit and FluentAssertions.
It’s just a matter of listing the attributes on the class, and checking that an attribute matching the one created exists.

[Fact]
public void Archives_controller_has_attribute()
{
    var attributes = typeof(ArchivesController).GetCustomAttributes(true);
    attributes.Count().Should().BeGreaterThan(0, "because there should be at least an attribute");
    var attribute = attributes.OfType<ApiKeyAuthorizeAttribute>().Single();
    attribute.Should().NotBeNull("because there should be an IApiKeyProvider on the archives controller");
}

[Fact]

public void Archives_controller_has_attribute()

{

var attributes = typeof(ArchivesController).GetCustomAttributes(true);

attributes.Count().Should().BeGreaterThan(0, "because there should be at least an attribute");

var attribute = attributes.OfType<ApiKeyAuthorizeAttribute>().Single();

attribute.Should().NotBeNull("because there should be an IApiKeyProvider on the archives controller");

}

Test the attribute inner workings

What are we testing there? That the attribute throws a HttpResponseException when no parameter exists or when the value is wrong, and that it doesn’t throw an exception when it matches.

Setup the tests

Using XUnit and Moq, the test setup looks like this:

private ApiKeyAuthorizeAttribute attribute = new ApiKeyAuthorizeAttribute();
private HttpActionContext context = ContextUtil.CreateActionContext();
private Mock<IStorageModel> storageModelMock = new Mock<IStorageModel>();
private Mock<IApiKeyProvider> apiKeyProviderMock = new Mock<IApiKeyProvider>();

public ApiKeyAuthorizeAttributeTests()
{
    apiKeyProviderMock.Setup(p => p.GetApiKey()).Returns("abc");
    storageModelMock.Setup(s => s.CreationArchived(It.IsAny<string>())).Returns(true);
    storageModelMock.Setup(s => s.CreationArchives(It.IsAny<string>())).Returns(new List<Archive>());
    storageModelMock.Setup(s => s.ListArchives(It.IsAny<int>(), It.IsAny<int>())).Returns(new List<Archive>());
    InjectionSetup.Register(context.ControllerContext.Configuration, storageModelMock.Object, apiKeyProviderMock.Object);
}

private ApiKeyAuthorizeAttribute attribute = new ApiKeyAuthorizeAttribute();

private HttpActionContext context = ContextUtil.CreateActionContext();

private Mock<IStorageModel> storageModelMock = new Mock<IStorageModel>();

private Mock<IApiKeyProvider> apiKeyProviderMock = new Mock<IApiKeyProvider>();

public ApiKeyAuthorizeAttributeTests()

{

apiKeyProviderMock.Setup(p => p.GetApiKey()).Returns("abc");

storageModelMock.Setup(s => s.CreationArchived(It.IsAny<string>())).Returns(true);

storageModelMock.Setup(s => s.CreationArchives(It.IsAny<string>())).Returns(new List<Archive>());

storageModelMock.Setup(s => s.ListArchives(It.IsAny<int>(), It.IsAny<int>())).Returns(new List<Archive>());

InjectionSetup.Register(context.ControllerContext.Configuration, storageModelMock.Object, apiKeyProviderMock.Object);

}

The ContextUtil.CreateActionContext method can be picked from the ASP.Net source. The corresponding tests can be found here.

My InjectionSetup.Register is a unit-test-specific injection that allows to use a specific instance instead of creating one, here using LightInject:

public static void Register(HttpConfiguration config, IStorageModel storageModel, IApiKeyProvider apiKeyProvider)
{
    var container = new ServiceContainer();
    container.Register<IStorageModel>((factory) => storageModel);
    container.Register<IApiKeyProvider>((factory) => apiKeyProvider);
    container.RegisterApiControllers();
    container.EnableWebApi(config);
}

public static void Register(HttpConfiguration config, IStorageModel storageModel, IApiKeyProvider apiKeyProvider)

{

var container = new ServiceContainer();

container.Register<IStorageModel>((factory) => storageModel);

container.Register<IApiKeyProvider>((factory) => apiKeyProvider);

container.RegisterApiControllers();

container.EnableWebApi(config);

}

Test the attribute

Still using XUnit and FluentAssertions, three simple tests allow to check that the responses are what is expected:

[Fact]
public void Missing_key_throws_error()
{
    context.Request.RequestUri = new Uri("http://localhost/api/archives");
    Action act = () => attribute.OnActionExecuting(context);
    act.ShouldThrow<HttpResponseException>("because a non-authorized action should throw an exception").And.Response.StatusCode.Should().Be(HttpStatusCode.Unauthorized, "because the request should not be authorized");
}

[Fact]
public void Wrong_key_throws_error()
{
    context.Request.RequestUri = new Uri("http://localhost/api/archives?api_key=wrong");
    Action act = () => attribute.OnActionExecuting(context);
    act.ShouldThrow<HttpResponseException>("because a wrong API key should throw an exception").And.Response.StatusCode.Should().Be(HttpStatusCode.Unauthorized, "because the request should not be authorized");
}

[Fact]
public void Right_key_passes()
{
    context.Request.RequestUri = new Uri("http://localhost/api/archives?api_key=abc");
    Action act = () => attribute.OnActionExecuting(context);
    act.ShouldNotThrow<HttpResponseException>("because an authorized action should not throw an exception");
}

[Fact]

public void Missing_key_throws_error()

{

context.Request.RequestUri = new Uri("http://localhost/api/archives");

Action act = () => attribute.OnActionExecuting(context);

act.ShouldThrow<HttpResponseException>("because a non-authorized action should throw an exception").And.Response.StatusCode.Should().Be(HttpStatusCode.Unauthorized, "because the request should not be authorized");

}

[Fact]

public void Wrong_key_throws_error()

{

context.Request.RequestUri = new Uri("http://localhost/api/archives?api_key=wrong");

Action act = () => attribute.OnActionExecuting(context);

act.ShouldThrow<HttpResponseException>("because a wrong API key should throw an exception").And.Response.StatusCode.Should().Be(HttpStatusCode.Unauthorized, "because the request should not be authorized");

}

[Fact]

public void Right_key_passes()

{

context.Request.RequestUri = new Uri("http://localhost/api/archives?api_key=abc");

Action act = () => attribute.OnActionExecuting(context);

act.ShouldNotThrow<HttpResponseException>("because an authorized action should not throw an exception");

}

Testing Entity Framework layer in database-first mode

In your app, you may have a “query” layer, where all your Linq queries are regrouped. If you don’t, you should. Having your queries inside your controller leads to poor testing and strong coupling.

You will want to test that your queries return what they say they return. In order to do that, you need to mock your Entity Framework entities container.

There is an awesome and complete tutorial here, explaining everything.

If you’re stuck on Visual Studio 2010, you will need to do a few things: first, download and install the ADO.NET DbContext Generator code template, so that your entities use the DbSet type. Then, open your Entity Framework container, right click inside and select “Add a code generation element” (or something like that, my VS is not in english) and select the DbContext elements you just downloaded. You will then be able to customize the way your entities are generated (awesome tool).

In all Visual Studio versions, if you’re not using code-first like the tutorial, you will have to modify the template generation to mark your entities list as virtual (so that they can be mocked). Open your xxx.Context.tt file, find the line with DbSet<<#=Code.Escape(entitySet.ElementType)#>> and add virtual in front of it. Now you can check out the generated xxx.Context.cs file to be sure it’s not doing crazy things.

While you’re at it, since you’re modifying the code templates, follow these awesome instructions to xml-document your generated entities.

Now, following the MSDN article, in your query tests (the ones returning a set of data), you will have to manually create the data to return, and “bind” it to the mock instance. Since you will have to do that in each of your query tests, don’t forget to extract it to a method:

/// <summary>
/// Sets up the mock set of entities to be used in tests
/// </summary>
/// <typeparam name="T">The type of the entities</typeparam>
/// <param name="mockSet">The mock set of entities to configure</param>
/// <param name="data">The data to use</param>
private void SetupMockEntities<T>(Mock mockSet, IQueryable<T> data)
{
    mockSet.As<IQueryable<T>>().Setup(m => m.Provider).Returns(data.Provider);
    mockSet.As<IQueryable<T>>().Setup(m => m.Expression).Returns(data.Expression);
    mockSet.As<IQueryable<T>>().Setup(m => m.ElementType).Returns(data.ElementType);
    mockSet.As<IQueryable<T>>().Setup(m => m.GetEnumerator()).Returns(data.GetEnumerator());
}

/// <summary>

/// Sets up the mock set of entities to be used in tests

/// </summary>

/// <typeparam name="T">The type of the entities</typeparam>

/// <param name="mockSet">The mock set of entities to configure</param>

/// <param name="data">The data to use</param>

private void SetupMockEntities<T>(Mock mockSet, IQueryable<T> data)

{

mockSet.As<IQueryable<T>>().Setup(m => m.Provider).Returns(data.Provider);

mockSet.As<IQueryable<T>>().Setup(m => m.Expression).Returns(data.Expression);

mockSet.As<IQueryable<T>>().Setup(m => m.ElementType).Returns(data.ElementType);

mockSet.As<IQueryable<T>>().Setup(m => m.GetEnumerator()).Returns(data.GetEnumerator());

}

Faking objects with FakeItEasy

Remember that time (yesterday) when I created my own fake implementations for my tests? Man, was I crazy.

There is an awesome library called FakeItEasy, which ease up your fake implementations.

Now, instead of implementing your own IEntitiesContainer (or whatever your data layer injection interface is), you can tell FakeItEasy to generate a new fake of this interface, and you get a bunch of assertion methods on it thrown in for good measure.

private readonly OrdersController controller;

private readonly IEntitiesContainer fakeContainer;

public CommandesControllerTests()
{
    // use a fake of the interface to inject in the controller
    this.fakeContainer = A.Fake<IEntitiesContainer>();
    this.controller = new OrdersController(this.fakeContainer);
}

[Fact]
public void Controller_gets_single_item()
{
    // arrange: the GetOrder method with whatever parameter will return something
    A.CallTo(() => this.fakeContainer.GetOrder(A<int>.Ignored)).Returns(new OrderDto());

    // act
    OrderDto result = this.controller.GetOrders(0);

    // assert: make sure the underlying IEntitiesContainer's proper method has been called
    result.Should().NotBeNull("because the method should fetch an item");
    A.CallTo(() => this.fakeContainer.GetOrder(A<int>.Ignored)).MustHaveHappened();
}

private readonly OrdersController controller;

private readonly IEntitiesContainer fakeContainer;

public CommandesControllerTests()

{

// use a fake of the interface to inject in the controller

this.fakeContainer = A.Fake<IEntitiesContainer>();

this.controller = new OrdersController(this.fakeContainer);

}

[Fact]

public void Controller_gets_single_item()

{

// arrange: the GetOrder method with whatever parameter will return something

A.CallTo(() => this.fakeContainer.GetOrder(A<int>.Ignored)).Returns(new OrderDto());

// act

OrderDto result = this.controller.GetOrders(0);

// assert: make sure the underlying IEntitiesContainer's proper method has been called

result.Should().NotBeNull("because the method should fetch an item");

A.CallTo(() => this.fakeContainer.GetOrder(A<int>.Ignored)).MustHaveHappened();

}

It also helps you make sure you’re testing the right things. For instance, I had a test method return a list of paginated result, and went to great lengths to make sure the proper number of items were returned. After spending a bit of time trying to refactor my test (because FakeItEasy doesn’t make it easy to provide specific parameters), I found out I don’t care about the number of returned results, since this is tested elsewhere.