Overriding the SystemColors, as most of the other answers advise, did not work for me. I ended up just editing the default style for ListBoxItem and setting ItemContainerStyle on the specific ListBox to the edited style. It's relatively painless compared to editing default styles for other controls. All you have to do is change the Item.SelectedInactive.Background and the Item.SelectedInactive.Border to the colors you want.
<SolidColorBrush x:Key="Item.Static.Background" Color="#FFFCFCFC" />
<SolidColorBrush x:Key="Item.Static.Border" Color="#FFFCFCFC" />
<SolidColorBrush x:Key="Item.MouseOver.Background" Color="#1F26A0DA" />
<SolidColorBrush x:Key="Item.MouseOver.Border" Color="#a826A0Da" />
<SolidColorBrush x:Key="Item.SelectedActive.Background" Color="#3D26A0DA" />
<SolidColorBrush x:Key="Item.SelectedActive.Border" Color="#FF26A0DA" />
<SolidColorBrush x:Key="Item.SelectedInactive.Background" Color="#3DDADADA" />
<SolidColorBrush x:Key="Item.SelectedInactive.Border" Color="#FFDADADA" />
<Style x:Key="ModifiedColorListBox" TargetType="{x:Type ListBoxItem}">
<Setter Property="SnapsToDevicePixels" Value="True" />
<Setter Property="Padding" Value="4,1" />
<Setter Property="HorizontalContentAlignment" Value="{Binding HorizontalContentAlignment, RelativeSource={RelativeSource AncestorType={x:Type ItemsControl}}}"/>
<Setter Property="VerticalContentAlignment" Value="{Binding VerticalContentAlignment, RelativeSource={RelativeSource AncestorType={x:Type ItemsControl}}}"/>
<Setter Property="Background" Value="Transparent" />
<Setter Property="BorderBrush" Value="Transparent" />
<Setter Property="BorderThickness" Value="1" />
<Setter Property="FocusVisualStyle" Value="{StaticResource FocusVisual}" />
<Setter Property="Template">
<Setter.Value>
<ControlTemplate TargetType="{x:Type ListBoxItem}">
<Border x:Name="Bd"
BorderBrush="{TemplateBinding BorderBrush}"
BorderThickness="{TemplateBinding BorderThickness}"
Background="{TemplateBinding Background}"
Padding="{TemplateBinding Padding}"
SnapsToDevicePixels="true">
<ContentPresenter
HorizontalAlignment="{TemplateBinding HorizontalContentAlignment}"
VerticalAlignment="{TemplateBinding VerticalContentAlignment}"
SnapsToDevicePixels="{TemplateBinding SnapsToDevicePixels}" />
</Border>
<ControlTemplate.Triggers>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="IsMouseOver" Value="True" />
</MultiTrigger.Conditions>
<Setter TargetName="Bd" Property="Background" Value="{StaticResource Item.MouseOver.Background}" />
<Setter TargetName="Bd" Property="BorderBrush" Value="{StaticResource Item.MouseOver.Border}" />
</MultiTrigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="Selector.IsSelectionActive" Value="False" />
<Condition Property="IsSelected" Value="True" />
</MultiTrigger.Conditions>
<Setter TargetName="Bd" Property="Background" Value="{StaticResource Item.SelectedInactive.Background}" />
<Setter TargetName="Bd" Property="BorderBrush" Value="{StaticResource Item.SelectedInactive.Border}" />
</MultiTrigger>
<MultiTrigger>
<MultiTrigger.Conditions>
<Condition Property="Selector.IsSelectionActive" Value="True" />
<Condition Property="IsSelected" Value="True" />
</MultiTrigger.Conditions>
<Setter TargetName="Bd" Property="Background" Value="{StaticResource Item.SelectedActive.Background}" />
<Setter TargetName="Bd" Property="BorderBrush" Value="{StaticResource Item.SelectedActive.Border}" />
</MultiTrigger>
<Trigger Property="IsEnabled" Value="False">
<Setter TargetName="Bd" Property="TextElement.Foreground" Value="{DynamicResource {x:Static SystemColors.GrayTextBrushKey}}"/>
</Trigger>
</ControlTemplate.Triggers>
</ControlTemplate>
</Setter.Value>
</Setter>
</Style>
And then usage from xaml:
<ListBox
Name="SomeListBox"
ItemContainerStyle="{StaticResource ModifiedColorListBox}"
ItemsSource="{Binding SomeCollection}"
SelectedItem="{Binding SomeObject}">
</ListBox>