Using PowerShell I needed to append a group to a Person or Group field in a list.  The same logic should apply for adding or removing a user.

Append Group:

$web = Get-SPweb "http://sharepointed.com/sites/taco/"
$list = $web.lists["Good Tacos"]

$groupName = "Taco Eaters"
$group = $web.SiteGroups[$groupName]
$GroupValue = new-Object Microsoft.SharePoint.SPFieldUserValue($web,$group.id, $group.Name)

foreach ($item in $list.items)
 {
 $groups = $item["GroupsField"]

$groups.Add($GroupValue)

$item["GroupsField"] = $groups
 $item.Update()
 }

Remove Group:

$web = Get-SPweb "http://sharepointed.com/sites/taco/"
$list = $web.lists["Good Tacos"]

$groupName = "Taco Eaters"
$group = $web.SiteGroups[$groupName]
$GroupValue = new-Object Microsoft.SharePoint.SPFieldUserValue($web,$group.id, $group.Name)

foreach ($item in $list.items)
{
$groups = $item["GroupsField"]

<span style="line-height: 1.5em;">$groupRemove = $groups | ? { $_.LookupId -eq $GroupValue.LookupId }</span>

$groups.Remove($groupRemove)

$item["GroupsField"] = $groups
$item.Update()
}