Django Management Command to List, Filter and Exclude Models from a Fixture

Dumping Django data to a fixture and loading it back up again are accomplished with the built-in management commands dumpdata and loaddata. However, loading a fixture into an existing database is a little trickier than loading it into an empty database. Because the fixture json contains the original primary keys of the records, you can get integrity errors.

Recently, I had a use case where I wanted to recover from this situation by excluding some models from the fixture. What I came up with is a new management command called copydata that takes an existing fixture file and can list, filter and exclude a subset of the models.

from django.core.management.base import BaseCommand
from optparse import make_option
import json


class Command(BaseCommand):

    operation = None
    filters = []
    excludes = []

    args = '<file [--list|--filter|--exclude]>'
    help = 'Copy/filter JSON fixture data to strip out certain models. ' \
        'Useful if certain parts of a fixture are failing.'

    option_list = BaseCommand.option_list + (
            make_option('--list',
                action='store_true',
                dest='list',
                default=False,
                help='List the models in a fixture'),
            make_option('--filter',
                action='store',
                dest='filter',
                default=False,
                help='Output the fixture with only these models, ' \
                    'comma-separated list.'),
            make_option('--exclude',
                action='store',
                dest='exclude',
                default=False,
                help='Output the fixture without these models, ' \
                    'comma-separated list.'),
            )

    def handle(self, file_path=None, list=False, filter=None, exclude=None, **options):

        if list:
            self.operation = 'list'
        if filter:
            self.filters = filter.split(',')
            self.operation = 'copy'
        if exclude:
            self.excludes = exclude.split(',')
            self.operation = 'copy'

        self.call('init')
        for record in json.load(open(file_path)):
            self.call('iter', record)
        self.call('final')

    def call(self, func_postfix, *args, **kwargs):
        func_name = self.operation + '_' + func_postfix
        if hasattr(self, func_name):
            func = getattr(self, func_name)
            func(*args, **kwargs)

    def list_init(self):
        self.model_set = set()

    def list_iter(self, record):
        self.model_set.add(record.get('model'))

    def list_final(self):
        for model in list(self.model_set):
            print model

    def copy_init(self):
        self.json_out = []

    def copy_iter(self, record):

        if self.filters and record.get('model') not in self.filters:
            return

        if self.excludes and record.get('model') in self.excludes:
            return

        self.json_out.append(record)

    def copy_final(self):
        print json.dumps(self.json_out, indent=4)

After saving this file as copydata.py inside the management/commands directory of your Django application, you can do the following.

>./manage.py dumpdata myapp > /tmp/fixture.json

>./manage.py copydata /tmp/fixture.json --list
myapp.friend
myapp.invite
myapp.profile

>./manage.py copydata /tmp/fixture.json --filter myapp.friend > /tmp/just_friends.json

>./manage.py copydata /tmp/fixture.json --exclude myapp.friend > /tmp/just_invites_and_profiles.json

If you still have access to the database where the fixture was dumped from, it's more straight forward to just dump the data again and use the built-in appname.Model arguments on the command-line, though you will need to list ALL of them in the exclude case.