No Django, existe uma maneira de filtrar em um campo manytomany vazio ou nulo.
class TestModel(models.Model):
name = models.CharField(_('set name'), max_length=200)
manytomany = models.ManyToManyField('AnotherModel', blank=True, null=True)
print TestModel.objects.filter(manytomany__is_null=True)
TestModel.objects.exclude(manytomany=None)